Skip to content

Feature: Extend Search API to include isLinked #12485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stevenwinship wants to merge 1 commit into
base: develop
Choose a base branch
from
+130 −185
Open

Feature: Extend Search API to include isLinked #12485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions conf/solr/schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@
<field name="fileRestricted" type="boolean" stored="true" indexed="false" multiValued="false"/>
<field name="canDownloadFile" type="boolean" stored="true" indexed="false" multiValued="false"/>
<field name="versionNote" type="string" stored="true" indexed="false" multiValued="false"/>
<field name="isLinked" type="boolean" stored="true" indexed="false" multiValued="false"/>

<!-- Added for Dataverse 4.0 alpha 1. This is a required field so we don't have to go to the database to get the database id of the entity. On cards we use the id in links -->
<field name="entityId" type="plong" stored="true" indexed="true" multiValued="false"/>
Expand Down
2 changes: 2 additions & 0 deletions doc/release-notes/11845-extend-search-api-return-islinked.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
## Feature Request ##
Search API now returns "isLinked" in the JSON response for datasets and collections that are linked. "isLinked" will also be returned in direct dataset and collection lookups. This attribute will only be included when the value is true.
43 changes: 13 additions & 30 deletions src/main/java/edu/harvard/iq/dataverse/Dataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,17 @@
import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations;
import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics;
import edu.harvard.iq.dataverse.settings.FeatureFlags;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.ColumnResult;
import jakarta.persistence.Entity;
import jakarta.persistence.Index;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.NamedNativeQuery;
import jakarta.persistence.NamedQueries;
import jakarta.persistence.NamedQuery;
import jakarta.persistence.OneToMany;
import jakarta.persistence.OneToOne;
import jakarta.persistence.OrderBy;
import jakarta.persistence.SqlResultSetMapping;
import jakarta.persistence.Table;
import jakarta.persistence.Temporal;
import jakarta.persistence.TemporalType;

import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.storageuse.StorageUse;
import edu.harvard.iq.dataverse.util.StringUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
import jakarta.persistence.*;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.*;

/**
*
Expand Down Expand Up @@ -810,13 +789,17 @@ public HarvestingClient getHarvestedFrom() {
public void setHarvestedFrom(HarvestingClient harvestingClientConfig) {
this.harvestedFrom = harvestingClientConfig;
}

public boolean isHarvested() {
return this.harvestedFrom != null;
}

public boolean isLinked() {
return (datasetLinkingDataverses != null && datasetLinkingDataverses.size() > 0);
}

private String harvestIdentifier;

public String getHarvestIdentifier() {
return harvestIdentifier;
}
Expand Down
42 changes: 9 additions & 33 deletions src/main/java/edu/harvard/iq/dataverse/Dataverse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,16 @@
package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem;
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.authorization.DataverseRole;
import edu.harvard.iq.dataverse.dataset.DatasetType;
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch;
import edu.harvard.iq.dataverse.storageuse.StorageUse;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
import jakarta.persistence.*;
import jakarta.validation.constraints.*;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.CollectionTable;
import jakarta.persistence.Column;
import jakarta.persistence.ElementCollection;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.Index;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.JoinTable;
import jakarta.persistence.ManyToMany;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.NamedQueries;
import jakarta.persistence.NamedQuery;
import jakarta.persistence.OneToMany;
import jakarta.persistence.OneToOne;
import jakarta.persistence.OrderBy;
import jakarta.persistence.Table;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Pattern;
import jakarta.validation.constraints.Size;
import java.util.*;

/**
*
Expand Down Expand Up @@ -327,7 +299,11 @@ public List<DatasetLinkingDataverse> getDatasetLinkingDataverses() {
public void setDatasetLinkingDataverses(List<DatasetLinkingDataverse> datasetLinkingDataverses) {
this.datasetLinkingDataverses = datasetLinkingDataverses;
}


public boolean isLinked() {
return (dataverseLinkingDataverses != null && dataverseLinkingDataverses.size() > 0);
}

public Set<ControlledVocabularyValue> getDataverseSubjects() {
return dataverseSubjects;
}
Expand Down
100 changes: 29 additions & 71 deletions src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,8 @@
package edu.harvard.iq.dataverse.search;

import edu.harvard.iq.dataverse.ControlledVocabularyValue;
import edu.harvard.iq.dataverse.CurationStatus;
import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.DataFileTag;
import edu.harvard.iq.dataverse.DataTable;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
import edu.harvard.iq.dataverse.DatasetFieldConstant;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.DatasetFieldValue;
import edu.harvard.iq.dataverse.DatasetFieldValueValidator;
import edu.harvard.iq.dataverse.DatasetLinkingServiceBean;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.*;
import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseLinkingServiceBean;
import edu.harvard.iq.dataverse.DataverseServiceBean;
import edu.harvard.iq.dataverse.DvObject;
import edu.harvard.iq.dataverse.DvObject.DType;
import edu.harvard.iq.dataverse.DvObjectServiceBean;
import edu.harvard.iq.dataverse.Embargo;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.GlobalId;
import edu.harvard.iq.dataverse.PermissionServiceBean;
import edu.harvard.iq.dataverse.Retention;
import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
Expand All @@ -50,51 +21,12 @@
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.StringUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import jakarta.ejb.AsyncResult;
import jakarta.ejb.Asynchronous;
import jakarta.ejb.EJB;
import jakarta.ejb.EJBException;
import jakarta.ejb.Stateless;
import jakarta.ejb.TransactionAttribute;

import static jakarta.ejb.TransactionAttributeType.REQUIRES_NEW;

import jakarta.ejb.*;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.json.JsonObject;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import jakarta.persistence.Query;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
Expand All @@ -107,15 +39,36 @@
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CursorMarkParams;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BodyContentHandler;
import org.eclipse.microprofile.metrics.MetricUnits;
import org.eclipse.microprofile.metrics.Timer;
import org.eclipse.microprofile.metrics.annotation.Metric;
import org.xml.sax.ContentHandler;

import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static jakarta.ejb.TransactionAttributeType.REQUIRES_NEW;

@Stateless
@Named
public class IndexServiceBean {
Expand Down Expand Up @@ -248,6 +201,9 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
solrInputDocument.addField(SearchFields.METADATA_SOURCE, rootDataverse.getName()); //rootDataverseName);
/*}*/

solrInputDocument.addField(SearchFields.IS_LINKED, dataverse.isLinked());


addDataverseReleaseDateToSolrDoc(solrInputDocument, dataverse);
// if (dataverse.getOwner() != null) {
// solrInputDocument.addField(SearchFields.HOST_DATAVERSE,
Expand Down Expand Up @@ -1063,6 +1019,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
solrInputDocument.addField(SearchFields.METADATA_SOURCE, rdvName); //rootDataverseName);
}

solrInputDocument.addField(SearchFields.IS_LINKED, dataset.isLinked());

DatasetType datasetType = dataset.getDatasetType();
solrInputDocument.addField(SearchFields.DATASET_TYPE, datasetType.getName());

Expand Down
3 changes: 3 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ public class SearchFields {
* Internal boolean used when creating OAI sets, for example.
*/
public static final String IS_HARVESTED = "isHarvested";

public static final String IS_LINKED = "isLinked";

/**
* Such as https://doi.org/10.5072/FK2/HXI35W
*
Expand Down
44 changes: 24 additions & 20 deletions src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,23 @@
package edu.harvard.iq.dataverse.search;

import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import java.util.logging.Logger;

import edu.harvard.iq.dataverse.*;
import jakarta.json.Json;
import jakarta.json.JsonArrayBuilder;
import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;

import org.apache.commons.collections4.CollectionUtils;

import edu.harvard.iq.dataverse.api.Util;
import edu.harvard.iq.dataverse.dataset.DatasetThumbnail;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.util.DateUtil;
import edu.harvard.iq.dataverse.util.json.JsonPrinter;
import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
import jakarta.json.Json;
import jakarta.json.JsonArrayBuilder;
import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;
import org.apache.commons.collections4.CollectionUtils;

import javax.xml.crypto.Data;
import java.util.*;
import java.util.function.Predicate;
import java.util.logging.Logger;

import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;

public class SolrSearchResult {

Expand Down Expand Up @@ -109,6 +101,8 @@ public class SolrSearchResult {
private String dataverseParentAlias;
private String dataverseParentName;
private List<Dataverse> collections;
private Boolean isLinked;

// private boolean statePublished;
/**
* @todo Investigate/remove this "unpublishedState" variable. For files that
Expand Down Expand Up @@ -582,9 +576,10 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool
.add("restricted", this.fileRestricted)
.add("variables", this.tabularDataCount)
.add("observations", this.observations)
.add("canDownloadFile", this.canDownloadFile);
.add("canDownloadFile", this.canDownloadFile)
.add("isLinked", this.isLinked);

// Now that nullSafeJsonBuilder has been instatiated, check for null before adding to it!
// Now that nullSafeJsonBuilder has been instantiated, check for null before adding to it!
if (showRelevance) {
nullSafeJsonBuilder.add("matches", getRelevance());
nullSafeJsonBuilder.add("score", getScore());
Expand Down Expand Up @@ -1135,6 +1130,15 @@ public Boolean getFileRestricted() {
public void setFileRestricted(Boolean fileRestricted) {
this.fileRestricted = fileRestricted;
}

public Boolean isLinked() {
return isLinked;
}

public void setLinked(Boolean isLinked) {
this.isLinked = isLinked;
}

public Boolean getCanDownloadFile() {
return canDownloadFile;
}
Expand Down
Loading
Loading