Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
83 commits
Select commit Hold shift + click to select a range
8246000
chore(deps): update jacksonversion to v2.19.0
renovate[bot] Apr 25, 2025
5ee65fd
chore(deps): update dependency gradle to v8.14
renovate[bot] Apr 25, 2025
f1a1a72
chore(deps): update dependency gg.jte:jte to v3.2.1
renovate[bot] Apr 25, 2025
02fe2c8
docs: convert minimal example from README to a test
Pfeil May 2, 2025
d43edf7
fix: avoid adding the same id property multiple times
Pfeil May 2, 2025
ac06985
cleanup: clean up unused imports and improve code formatting
Pfeil May 2, 2025
e9a3bd5
ci: always run CI on pull requests
Pfeil May 2, 2025
b167bae
Merge pull request #257 from kit-data-manager/fix-adding-id-property-…
Pfeil May 2, 2025
d47f90d
refactor(tests): factor out helper methods
Pfeil May 2, 2025
0c43211
docs: update examples in ExamplesOfSpecificationV1p1Test with reposit…
Pfeil May 2, 2025
424b8f6
chore: improve readability of test to make it more of a guide
Pfeil May 2, 2025
375e53a
docs: add javadocs to RoCrateBuilder.addIdentifier
Pfeil May 2, 2025
27d671e
docs: convert files-and-folders example from README to a test
Pfeil May 2, 2025
f24a7a6
docs: enhance Javadoc for id property in AbstractEntity to clarify ID…
Pfeil May 2, 2025
917b89f
fix: javadoc compilation with java 17 does not support @apiNote
Pfeil May 2, 2025
b551a8c
docs: enhance Javadoc in ExamplesOfSpecificationV1p1Test to introduce…
Pfeil May 2, 2025
d35c176
docs: convert web-based-data-entities example from README to a test
Pfeil May 2, 2025
fe02a9f
docs: fix example description for web-based data entities in Examples…
Pfeil May 2, 2025
8ab6f1b
docs: convert file-author-location example from README to a test
Pfeil May 2, 2025
8197ce8
fix: file-author-location test is missing license and datePublished.
Pfeil May 2, 2025
a8e8e2e
docs: add complete workflow example test
Pfeil May 2, 2025
3eec23d
docs: refactor specification examples section in README into other ex…
Pfeil May 2, 2025
61fd863
docs: move quickstart to LearnByExampleTest so it is always tested pr…
Pfeil May 3, 2025
810d199
Merge pull request #256 from kit-data-manager/make-spec-examples-exec…
Pfeil May 5, 2025
cdecbed
test: add parameterized test for reading ELN crates from URLs
Pfeil May 5, 2025
c7e0afc
refactor: enable readers to implement subsets of tests if they make s…
Pfeil May 5, 2025
2e0df35
feat: reading zip files now searches for a crate in subdirectories.
Pfeil May 5, 2025
ccc0fe8
chore: fix deprecation warning in ElnFileFormatTest and enable better…
Pfeil May 5, 2025
0434a73
fix: improve error reporting for JSON schema validation
Pfeil May 5, 2025
966255f
refactor: replace deprecated setAll method with setAllIfValid and set…
Pfeil May 5, 2025
d0c1103
refactor: use try-with-resources for CloseableHttpClient in RoCrateMe…
Pfeil May 5, 2025
cdb897c
fix: correct typos and improve formatting in documentation and error …
Pfeil May 5, 2025
160ba97
docs: enhance documentation and add tests for setAllIfValid and setAl…
Pfeil May 5, 2025
86654f6
fix: invalid entities from our providers should be logged and not wri…
Pfeil May 5, 2025
7e4a30f
refactor: optimize file search in ZipStrategy by using directory filter
Pfeil May 6, 2025
49f1b2c
fix: improve metadata file resolution in ZipStreamStrategy to search …
Pfeil May 6, 2025
6f1948c
fix: handle directory entries in ZipStreamStrategy to ensure proper e…
Pfeil May 6, 2025
5dd5421
fix: integrate Apache Commons Compress for improved zip file handling…
Pfeil May 6, 2025
a704a0b
Revert "fix: integrate Apache Commons Compress for improved zip file …
Pfeil May 8, 2025
586276b
Revert "fix: invalid entities from our providers should be logged and…
Pfeil May 8, 2025
eac3a75
fix: propagate IOException in crate reading and writing methods
Pfeil May 8, 2025
62b1b0c
refactor: setAll method now uses setAllIfValid to avoid code duplication
Pfeil May 8, 2025
d3f7986
tests: add blacklist functionality for unsupported ELN files in tests
Pfeil May 8, 2025
cca2cd5
tests: remove possible side effects between tests in LearnByExampleTe…
Pfeil May 8, 2025
6fd9ac2
refactor: pull DataEntity saveToStream logic into writer.ZipStreamStr…
Pfeil May 8, 2025
b58d6bd
refactor: propagate IOExceptions properly in writer.ZipStreamStrategy…
Pfeil May 8, 2025
6e1dae6
refactor: propagate IOExceptions properly in writer.FolderStrategy
Pfeil May 8, 2025
82cee04
refactor: propagate IOExceptions properly in writer.ZipStrategy
Pfeil May 8, 2025
bee2917
refactor: pull DataEntity saveToZip logic into writer.ZipStrategy
Pfeil May 8, 2025
08a5f10
refactor: remove outdated saveToStream method documentation in writer…
Pfeil May 8, 2025
0fc94e7
refactor: move savetoFile logic from DataEntity to FolderStrategy
Pfeil May 8, 2025
e41cfcf
refactor: propagate IOException handling in reader.FolderStrategy and…
Pfeil May 8, 2025
bccbf0b
tests: implement ELN style crate writer test for ZipWriterTest
Pfeil May 9, 2025
b46be7f
feat: implement eln style writing for reader.ZipStrategy
Pfeil May 9, 2025
a296f91
test: add test for withRootSubdirectory
Pfeil May 9, 2025
b25d1eb
fix: remove IOException documentation from usingElnStyle and withRoot…
Pfeil May 9, 2025
a09d6f0
docs: add return documentation for usingElnStyle and withRootSubdirec…
Pfeil May 9, 2025
00ee66e
test: share alias test with all eln writers and implement test for Zi…
Pfeil May 12, 2025
e2a4bf3
feat: support ELN format in ZipStreamStrategy
Pfeil May 12, 2025
8455f4f
test: add tests for saving previews to ZipOutputStream in PreviewTest
Pfeil May 19, 2025
5ef25d4
fix: enforce correct subfolder name for additional files in StaticPre…
Pfeil May 19, 2025
e4d91d0
Merge pull request #261 from kit-data-manager/fix-preview-additional-…
Pfeil May 19, 2025
cfa6f73
refactor: prefix reader strategies and writer strategies with Read or…
Pfeil May 19, 2025
92ce1cb
refactor: remove unused import in ZipWriterTest
Pfeil May 19, 2025
87f790b
refactor: rename ZipStreamStrategyTest to ZipStreamWriterTest and upd…
Pfeil May 19, 2025
b22e70b
docs: Improve reader and writer documentation, and related type param…
Pfeil May 19, 2025
896018d
refactor: started sharing code between WriteZipStrategy and WriteZipS…
Pfeil May 19, 2025
fd9b84d
refactor: factor out common tmp directory handling in zip readers
Pfeil May 19, 2025
1e2da11
refactor: simplify WriteZipStrategy by delegating save operations to …
Pfeil May 19, 2025
9cf26b1
docs: add IOException documentation to readCrate method and clarify C…
Pfeil May 19, 2025
06b2146
test: add not-null-assertions for crate and person entity
Pfeil May 19, 2025
6c4c182
Merge pull request #258 from kit-data-manager/support-eln-files
Pfeil May 19, 2025
01058be
refactor: improve null safety in mkdirOrDeleteContent by using local …
Pfeil May 19, 2025
7263e78
docs: update filterExtensionsFromFileName method documentation to spe…
Pfeil May 19, 2025
fea8452
refactor: use try-with-resources for FileOutputStream in saveCrate me…
Pfeil May 19, 2025
2de9b63
fix: log error when failing to delete temporary files in CratePreview
Pfeil May 19, 2025
a6ad7d6
fix: ensure extracted files are within the target directory using rea…
Pfeil May 19, 2025
3db4945
refactor: update path handling in addFolderToZipStream to use File ob…
Pfeil May 19, 2025
d3336ff
refactor: use try-with-resources for InputStream and ZipOutputStream …
Pfeil May 19, 2025
289ffee
fix: increase timeout for downloading ELN file in ElnFileFormatTest
Pfeil May 21, 2025
d43a8dd
refactor: use try-with-resources for FileInputStream and FileOutputSt…
Pfeil May 21, 2025
684734b
refactor: ensure inconsistencies are being fixed when adding an id pr…
Pfeil May 21, 2025
ee9bdb1
refactor: change mergeIdIntoValue method visibility to protected and …
Pfeil May 21, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ on:
push:
branches: [ main, development ]
pull_request:
branches: [ main, development ]
workflow_dispatch:

env:
Expand Down
691 changes: 19 additions & 672 deletions README.md

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ repositories {
}

ext {
jacksonVersion = '2.18.3'
jacksonVersion = '2.19.0'
}

dependencies {
Expand Down Expand Up @@ -67,10 +67,15 @@ dependencies {
implementation group: "com.networknt", name: "json-schema-validator", version: "1.5.6"
implementation 'org.glassfish:jakarta.json:2.0.1'
//JTE for template processing
implementation('gg.jte:jte:3.2.0')
implementation('gg.jte:jte:3.2.1')
implementation("org.freemarker:freemarker:2.3.34")
}

// enable -Xlint:deprecation
tasks.withType(JavaCompile).configureEach {
options.compilerArgs << "-Xlint:deprecation"
}

logging.captureStandardOutput LogLevel.INFO

def signingTasks = tasks.withType(Sign)
Expand Down
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
Expand Down
4 changes: 2 additions & 2 deletions gradlew

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions gradlew.bat

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 13 additions & 7 deletions src/main/java/edu/kit/datamanager/ro_crate/RoCrate.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,10 @@
import edu.kit.datamanager.ro_crate.entities.AbstractEntity;
import edu.kit.datamanager.ro_crate.entities.contextual.ContextualEntity;
import edu.kit.datamanager.ro_crate.entities.contextual.JsonDescriptor;
import edu.kit.datamanager.ro_crate.entities.contextual.OrganizationEntity;
import edu.kit.datamanager.ro_crate.entities.data.DataEntity;
import edu.kit.datamanager.ro_crate.entities.data.DataEntity.DataEntityBuilder;
import edu.kit.datamanager.ro_crate.entities.data.FileEntity;

import edu.kit.datamanager.ro_crate.entities.data.RootDataEntity;
import edu.kit.datamanager.ro_crate.externalproviders.dataentities.ImportFromDataCite;
import edu.kit.datamanager.ro_crate.externalproviders.organizationprovider.RorProvider;
import edu.kit.datamanager.ro_crate.objectmapper.MyObjectMapper;
import edu.kit.datamanager.ro_crate.payload.CratePayload;
import edu.kit.datamanager.ro_crate.payload.RoCratePayload;
Expand All @@ -26,12 +23,9 @@
import edu.kit.datamanager.ro_crate.special.JsonUtilFunctions;
import edu.kit.datamanager.ro_crate.validation.JsonSchemaValidation;
import edu.kit.datamanager.ro_crate.validation.Validator;
import edu.kit.datamanager.ro_crate.writer.FolderWriter;
import edu.kit.datamanager.ro_crate.writer.RoCrateWriter;

import java.io.File;
import java.net.URI;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
Expand Down Expand Up @@ -354,6 +348,18 @@ public RoCrateBuilder addName(String name) {
return this;
}

/**
* Adds an "identifier" property to the root data entity.
* <p>
* This is useful e.g. to assign e.g. a DOI to this crate.
* @param identifier the identifier to add.
* @return this builder.
*/
public RoCrateBuilder addIdentifier(String identifier) {
this.rootDataEntity.addProperty("identifier", identifier.strip());
return this;
}

public RoCrateBuilder addDescription(String description) {
this.rootDataEntity.addProperty(PROPERTY_DESCRIPTION, description);
return this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,10 @@ public boolean checkEntity(AbstractEntity entity) {
node.remove("@id");
node.remove("@type");

Set<String> types = objectMapper.convertValue(entity.getProperties().get("@type"),
new TypeReference<>() {
});
Set<String> types = objectMapper.convertValue(
entity.getProperties().path("@type"),
new TypeReference<>() {}
);
// check if the items in the array of types are present in the context
for (String s : types) {
// special cases:
Expand Down Expand Up @@ -174,15 +175,14 @@ public void addToContextFromUrl(String url) {
}
}
if (jsonNode == null) {
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
CloseableHttpResponse response;
try {
try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
response = httpclient.execute(httpGet);
jsonNode = objectMapper.readValue(response.getEntity().getContent(),
JsonNode.class);
} catch (IOException e) {
System.err.println(String.format("Cannot get context from url %s", url));
System.err.printf("Cannot get context from url %s%n", url);
return;
}
if (url.equals(DEFAULT_CONTEXT)) {
Expand Down
139 changes: 107 additions & 32 deletions src/main/java/edu/kit/datamanager/ro_crate/entities/AbstractEntity.java
Original file line number Diff line number Diff line change
Expand Up @@ -239,33 +239,61 @@ private static boolean addProperty(ObjectNode whereToAdd, String key, JsonNode v
* @param id the "id" of the property.
*/
public void addIdProperty(String name, String id) {
JsonNode jsonNode = addToIdProperty(name, id, this.properties.get(name));
if (jsonNode != null) {
this.linkedTo.add(id);
this.properties.set(name, jsonNode);
this.notifyObservers();
}
if (id == null || id.isBlank()) { return; }
mergeIdIntoValue(id, this.properties.get(name))
.ifPresent(newValue -> {
this.properties.set(name, newValue);
});
this.linkedTo.add(id);
this.notifyObservers();
}

private static JsonNode addToIdProperty(String name, String id, JsonNode property) {
ObjectMapper objectMapper = MyObjectMapper.getMapper();
if (name != null && id != null) {
if (property == null) {
return objectMapper.createObjectNode().put("@id", id);
} else {
if (property.isArray()) {
ArrayNode ns = (ArrayNode) property;
ns.add(objectMapper.createObjectNode().put("@id", id));
return ns;
} else {
ArrayNode newNodes = objectMapper.createArrayNode();
newNodes.add(property);
newNodes.add(objectMapper.createObjectNode().put("@id", id));
return newNodes;
}
}
/**
* Merges the given id into the current value,
* using this representation: {"@id" : "id"}.
* <p>
* The current value can be null without errors.
* Only the id will be considered in this case.
* <p>
* If the id is null-ish, it will not be added, similar to a null-ish value.
* If the id is already present, nothing will be done.
* If it is not an array and the id is not present, an array will be applied.
*
* @param id the id to add.
* @param currentValue the current value of the property.
* @return The updated value of the property.
* Empty if value does not change!
*/
protected static Optional<JsonNode> mergeIdIntoValue(String id, JsonNode currentValue) {
if (id == null || id.isBlank()) { return Optional.empty(); }

ObjectMapper jsonBuilder = MyObjectMapper.getMapper();
ObjectNode newIdObject = jsonBuilder.createObjectNode().put("@id", id);
if (currentValue == null || currentValue.isNull() || currentValue.isMissingNode()) {
return Optional.ofNullable(newIdObject);
}

boolean isIdAlready = currentValue.asText().equals(id);
boolean isIdObjectAlready = currentValue.path("@id").asText().equals(id);
boolean isArrayWithIdPresent = currentValue.valueStream()
.anyMatch(node -> node
.path("@id")
.asText()
.equals(id));
if (isIdAlready || isIdObjectAlready || isArrayWithIdPresent) {
return Optional.empty();
}

if (currentValue.isArray() && currentValue instanceof ArrayNode currentValueAsArray) {
currentValueAsArray.add(newIdObject);
return Optional.of(currentValueAsArray);
} else {
// property is not an array, so we make it an array
ArrayNode newNodes = jsonBuilder.createArrayNode();
newNodes.add(currentValue);
newNodes.add(newIdObject);
return Optional.of(newNodes);
}
return null;
}

/**
Expand Down Expand Up @@ -369,6 +397,11 @@ protected String getId() {
/**
* Setting the id property of the entity, if the given value is not
* null. If the id is not encoded, the encoding will be done.
* <p>
* <b>NOTE: IDs are not just names!</b> The ID may have effects
* on parts of your crate! For example: If the entity represents a
* file which will be copied into the crate, writers must use the
* ID as filename.
*
* @param id the String representing the id.
* @return the generic builder.
Expand Down Expand Up @@ -486,11 +519,11 @@ public T addProperty(String key, boolean value) {
* @return the generic builder
*/
public T addIdProperty(String name, String id) {
JsonNode jsonNode = AbstractEntity.addToIdProperty(name, id, this.properties.get(name));
if (jsonNode != null) {
this.properties.set(name, jsonNode);
this.relatedItems.add(id);
}
AbstractEntity.mergeIdIntoValue(id, this.properties.get(name))
.ifPresent(newValue -> {
this.properties.set(name, newValue);
this.relatedItems.add(id);
});
return self();
}

Expand Down Expand Up @@ -526,20 +559,62 @@ public T addIdFromCollectionOfEntities(String name, Collection<AbstractEntity> e
}

/**
* This sets everything from a json object to the property. Can be
* useful when the entity is already available somewhere.
* Deprecated. Equivalent to {@link #setAllIfValid(ObjectNode)}.
*
* @param properties the Json representing all the properties.
* @return the generic builder.
* @return the generic builder, either including all given properties
* * or unchanged.
*
* @deprecated To enforce the user know what this method does,
* we want the user to use one of the more explicitly named
* methods {@link #setAllIfValid(ObjectNode)} or
* {@link #setAllIfValid(ObjectNode)}.
* @see #setAllIfValid(ObjectNode)
*/
@Deprecated(since = "2.1.0", forRemoval = true)
public T setAll(ObjectNode properties) {
return setAllIfValid(properties);
}

/**
* This sets everything from a json object to the property,
* <b>if the result is valid</b>. Otherwise, it will do <b>nothing</b>.
* <p>
* Valid means here that the json object needs to be flat as specified
* in the RO-Crate specification. In principle, this means that
* primitives and objects referencing an ID are allowed,
* as well as arrays of these.
*
* @param properties the Json representing all the properties.
* @return the generic builder, either including all given properties
* or unchanged.
*/
public T setAllIfValid(ObjectNode properties) {
if (AbstractEntity.entityValidation.entityValidation(properties)) {
this.properties = properties;
this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties));
}
return self();
}

/**
* This sets everything from a json object to the property. Can be
* useful when the entity is already available somewhere.
* <p>
* Errors on validation are printed, but everything will be added.
* For more about validation, see {@link #setAllIfValid(ObjectNode)}.
*
* @param properties the Json representing all the properties.
* @return the generic builder with all properties added.
*/
public T setAllUnsafe(ObjectNode properties) {
// This will currently only print errors.
AbstractEntity.entityValidation.entityValidation(properties);
this.properties = properties;
this.relatedItems.addAll(JsonUtilFunctions.getIdPropertiesFromJsonNode(properties));
return self();
}

public abstract T self();

public abstract AbstractEntity build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@

public class JsonDescriptor extends ContextualEntity {

private static final String CONFORMS_TO = "conformsTo";
protected static final String ID = "ro-crate-metadata.json";
protected static final String CONFORMS_TO = "conformsTo";
public static final String ID = "ro-crate-metadata.json";

/**
* Returns a JsonDescriptor with the conformsTo value set to the latest stable
Expand All @@ -39,7 +39,7 @@ private JsonDescriptor(ContextualEntityBuilder builder) {

/**
* Builder for the JsonDescriptor.
*
* <p>
* Defaults to the latest stable crate version and no other conformsTo values.
*/
public static final class Builder {
Expand Down
Loading