From 2423152203f654a07c1fe5ad92ed2a9803df2d1f Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 28 Apr 2025 14:51:47 -0700 Subject: [PATCH 1/3] Issue 51979: BadSqlGrammarException indexing sample types immediately after a field rename --- .../experiment/api/AbstractRunItemImpl.java | 148 ++++++++++++++++++ .../labkey/experiment/api/ExpDataImpl.java | 121 +------------- .../experiment/api/ExpMaterialImpl.java | 42 +++-- .../experiment/api/ExpMaterialTableImpl.java | 4 +- .../experiment/api/ExperimentServiceImpl.java | 2 +- .../experiment/api/SampleTypeServiceImpl.java | 2 +- .../api/SampleTypeUpdateServiceDI.java | 6 +- 7 files changed, 190 insertions(+), 135 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java index f8f2c99987d..5c40a01600c 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java @@ -16,9 +16,16 @@ package org.labkey.experiment.api; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; +import org.labkey.api.data.MultiValuedLookupColumn; +import org.labkey.api.data.MultiValuedRenderContext; +import org.labkey.api.data.Results; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlSelector; @@ -32,14 +39,23 @@ import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExpRunItem; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataClassDataTable; +import org.labkey.api.query.FieldKey; +import org.labkey.api.search.SearchService; import org.labkey.api.security.User; import org.labkey.api.security.UserManager; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Base class for both types of objects that can be the input and output from an experiment run - material and data. @@ -48,6 +64,8 @@ */ public abstract class AbstractRunItemImpl extends ExpIdentifiableBaseImpl implements ExpRunItem { + private static final Logger LOG = LogManager.getLogger(AbstractRunItemImpl.class); + private ExpProtocolApplicationImpl _sourceApp; private List _successorAppList; private List _successorRunIdList = null; @@ -315,4 +333,134 @@ protected HashMap getObjectProperties(TableInfo ti) return ret; }); } + + public void processIndexValues( + ExpRunItemTableImpl table, + CaseInsensitiveHashSet skipColumns, + Set identifiersHi, + Set identifiersMed, + Set identifiersLo, + Set keywordHi, + Set keywordMed, + Set keywordsLo, + JSONObject jsonData + ) + { + skipColumns.add("genId"); + skipColumns.add("rowId"); + // collect the set of columns to index + Set columns = table.getExtendedColumns(true).values().stream().filter(col -> { + // skip the base-columns - they will be added to the index separately and/or we don't want to index them (Issue 52467) + if (skipColumns.contains(col.getName())) + return false; + + // skip non-text and non-int columns or columns that aren't lookups + if (!(col.getJdbcType().isText() || col.getJdbcType().isInteger() || col.getFk() != null)) + return false; + + // Issue 52467: Skip indexing both the raw columns like LSID and the wrapped versions of those columns that are lookups to other data + if ("lsidtype".equalsIgnoreCase(col.getSqlTypeName()) || "entityid".equalsIgnoreCase(col.getSqlTypeName())) + return false; + + return true; + }).collect(Collectors.toCollection(LinkedHashSet::new)); + + if (columns.isEmpty()) + return; + + TableSelector ts = new TableSelector(table, columns, new SimpleFilter(ExpDataClassDataTable.Column.RowId.fieldKey(), getRowId()), null); + ts.setForDisplay(true); + try (Results r = ts.getResults()) + { + Map fields = r.getFieldMap(); + if (r.next()) + { + Map map = r.getFieldKeyRowMap(); + for (Map.Entry entry : fields.entrySet()) + { + FieldKey fieldKey = entry.getKey(); + ColumnInfo col = entry.getValue(); + if (!col.getJdbcType().isText() && !col.getJdbcType().isInteger()) + continue; + + if (col.getName().equalsIgnoreCase("lsid") || col.getSqlTypeName().equalsIgnoreCase("lsidtype") || col.getSqlTypeName().equalsIgnoreCase("entityid")) + continue; + + Object o = map.get(fieldKey); + String s; + // Issue 52961: DataClass: Integer fields are not index for data class + if (o instanceof String) + s = (String)o; + else if (o instanceof Integer) + s = String.valueOf(o); + else + continue; + + List values; + + if (col instanceof MultiValuedLookupColumn) + values = Arrays.asList(s.split(MultiValuedRenderContext.VALUE_DELIMITER_REGEX)); + else + values = Arrays.asList(s); + + SearchService.PROPERTY searchProperty = table.getSearchIndexColumn(fieldKey); + if (searchProperty != null) + { + // Fow now only add indexed field values to search jsonData + if (!values.isEmpty()) + { + if (values.size() == 1) + jsonData.put(fieldKey.toString(), values.get(0)); + else + jsonData.put(fieldKey.toString(), values); + } + + switch (searchProperty) + { + case identifiersHi -> { + identifiersHi.addAll(values); + continue; + } + case identifiersMed -> { + identifiersMed.addAll(values); + continue; + } + case identifiersLo -> { + identifiersLo.addAll(values); + continue; + } + case keywordsHi -> { + keywordHi.addAll(values); + continue; + } + case keywordsMed -> { + keywordMed.addAll(values); + continue; + } + case keywordsLo -> { + keywordsLo.addAll(values); + continue; + } + default -> LOG.debug("Unable to index column " + fieldKey.toString() + " with property: " + searchProperty.name() + ". Not yet supported."); + } + } + + if ("textarea".equalsIgnoreCase(col.getInputType())) + { + // treat multi-line text values as keywords, otherwise treat as an identifier + keywordsLo.addAll(values); + } + else + { + identifiersMed.addAll(values); + } + } + } + } + catch (SQLException e) + { + // ignore + } + + } } diff --git a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java index dc632162010..5ab764ee6cd 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java @@ -17,26 +17,19 @@ package org.labkey.experiment.api; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.MultiValuedLookupColumn; -import org.labkey.api.data.MultiValuedRenderContext; -import org.labkey.api.data.Results; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; import org.labkey.api.exp.ExperimentDataHandler; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Handler; @@ -91,15 +84,12 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; -import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -108,7 +98,6 @@ public class ExpDataImpl extends AbstractRunItemImpl implements ExpData { - private static final Logger LOG = LogManager.getLogger(ExpDataImpl.class); public enum DataOperations { Edit("editing", UpdatePermission.class), @@ -495,7 +484,7 @@ public void importDataFile(PipelineJob job, XarSource xarSource) throws Experime job.debug("Finished trying to load data file " + dataFileURL + " into the system"); } - // Get all text strings from the data class for indexing + // Get all text and int strings from the data class for indexing private void getIndexValues( ExpDataClassDataTableImpl table, Set identifiersHi, @@ -513,113 +502,7 @@ private void getIndexValues( skipColumns.add("Ancestors"); skipColumns.add("Container"); - // collect the set of columns to index - Set columns = table.getExtendedColumns(true).values().stream().filter(col -> { - // skip the base-columns - they will be added to the index separately and/or we don't want to index them (Issue 52467) - if (skipColumns.contains(col.getName())) - return false; - - // skip non-text columns or columns that aren't lookups - if (!(col.getJdbcType().isText() || col.getFk() != null)) - return false; - - // Issue 52467: Skip indexing both the raw columns like LSID and the wrapped versions of those columns that are lookups to other data - if ("lsidtype".equalsIgnoreCase(col.getSqlTypeName()) || "entityid".equalsIgnoreCase(col.getSqlTypeName())) - return false; - - return true; - }).collect(Collectors.toCollection(LinkedHashSet::new)); - - if (columns.isEmpty()) - return; - - TableSelector ts = new TableSelector(table, columns, new SimpleFilter(ExpDataClassDataTable.Column.RowId.fieldKey(), getRowId()), null); - ts.setForDisplay(true); - try (Results r = ts.getResults()) - { - Map fields = r.getFieldMap(); - if (r.next()) - { - Map map = r.getFieldKeyRowMap(); - for (Map.Entry entry : fields.entrySet()) - { - FieldKey fieldKey = entry.getKey(); - ColumnInfo col = entry.getValue(); - if (!col.getJdbcType().isText()) - continue; - - if (col.getName().equalsIgnoreCase("lsid") || col.getSqlTypeName().equalsIgnoreCase("lsidtype") || col.getSqlTypeName().equalsIgnoreCase("entityid")) - continue; - - Object o = map.get(fieldKey); - if (!(o instanceof String s)) - continue; - - List values; - - if (col instanceof MultiValuedLookupColumn) - values = Arrays.asList(s.split(MultiValuedRenderContext.VALUE_DELIMITER_REGEX)); - else - values = Arrays.asList(s); - - SearchService.PROPERTY searchProperty = table.getSearchIndexColumn(fieldKey); - if (searchProperty != null) - { - // Fow now only add indexed field values to search jsonData - if (!values.isEmpty()) - { - if (values.size() == 1) - jsonData.put(fieldKey.toString(), values.get(0)); - else - jsonData.put(fieldKey.toString(), values); - } - - switch (searchProperty) - { - case identifiersHi -> { - identifiersHi.addAll(values); - continue; - } - case identifiersMed -> { - identifiersMed.addAll(values); - continue; - } - case identifiersLo -> { - identifiersLo.addAll(values); - continue; - } - case keywordsHi -> { - keywordHi.addAll(values); - continue; - } - case keywordsMed -> { - keywordMed.addAll(values); - continue; - } - case keywordsLo -> { - keywordsLo.addAll(values); - continue; - } - default -> LOG.debug("Unable to index column " + fieldKey.toString() + " with property: " + searchProperty.name() + ". Not yet supported."); - } - } - - if ("textarea".equalsIgnoreCase(col.getInputType())) - { - // treat multi-line text values as keywords, otherwise treat as an identifier - keywordsLo.addAll(values); - } - else - { - identifiersMed.addAll(values); - } - } - } - } - catch (SQLException e) - { - // ignore - } + processIndexValues(table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); } @Override diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index b743eaf4b40..3ec6a1915fe 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -22,6 +22,8 @@ import org.apache.logging.log4j.Level; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -52,6 +54,7 @@ import org.labkey.api.qc.SampleStatusService; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryRowReference; +import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; import org.labkey.api.search.SearchService; import org.labkey.api.security.User; @@ -79,6 +82,7 @@ import java.util.Map; import java.util.Set; +import static org.labkey.api.exp.query.SamplesSchema.SCHEMA_SAMPLES; import static org.labkey.api.util.StringUtilsLabKey.append; public class ExpMaterialImpl extends AbstractRunItemImpl implements ExpMaterial @@ -311,7 +315,7 @@ public void save(User user, ExpSampleTypeImpl st) SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(st, SampleTypeServiceImpl.SampleChangeType.insert); } } - index(null); + index(null, null); } @Override @@ -381,7 +385,7 @@ public String getValue(ObjectProperty prop, List siblingProperti } }; - public void index(@Nullable SearchService.IndexTask task) + public void index(@Nullable SearchService.IndexTask task, @Nullable ExpMaterialTableImpl tableInfo) { // Big hack to prevent study specimens and bogus samples created from some plate assays (Issue 46037) // from being indexed as samples @@ -399,17 +403,16 @@ public void index(@Nullable SearchService.IndexTask task) // do the least possible amount of work here final SearchService.IndexTask indexTask = task; - var document = createIndexDocument(); + var document = createIndexDocument(tableInfo); if (document != null) { indexTask.addResource(document, SearchService.PRIORITY.item); } } - /** returns null if the parent container is no longer available */ @Nullable - public WebdavResource createIndexDocument() + public WebdavResource createIndexDocument(ExpMaterialTableImpl tableInfo) { Container container = getContainer(); if (container == null) @@ -438,7 +441,6 @@ public WebdavResource createIndexDocument() props.put(SearchService.PROPERTY.categories.toString(), searchCategory.toString()); props.put(SearchService.PROPERTY.title.toString(), title.toString()); props.put(SearchService.PROPERTY.keywordsLo.toString(), "Sample"); // Treat the word "Sample" a low priority keyword - props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); StringBuilder body = new StringBuilder(); @@ -449,11 +451,16 @@ public WebdavResource createIndexDocument() append(body, getSourceApplication()); // Add all String and Integer custom property descriptions and values to body - CustomProperties.iterate(getContainer(), getObjectProperties().values(), RENDERER_MAP, (indent, description, value) -> + if (null != getSampleType()) { - append(body, description); - append(body, value); - }); + if (tableInfo == null) + tableInfo = (ExpMaterialTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_SAMPLES).getTable(getSampleType().getName()); + + if (tableInfo != null) + getCustomIndexValues(tableInfo, identifiersHi, new JSONObject()); + } + + props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); ExpSampleType st = getSampleType(); if (null != st) @@ -505,6 +512,21 @@ public void setLastIndexed(long ms, long modified) }; } + // Get all text and int strings from the material properties for indexing + private void getCustomIndexValues( + ExpMaterialTableImpl table, + Set identifiersHi, + JSONObject jsonData + ) + { + CaseInsensitiveHashSet skipColumns = new CaseInsensitiveHashSet(); + for (ExpMaterialTable.Column column : ExpMaterialTable.Column.values()) + skipColumns.add(column.name()); + + processIndexValues(table, skipColumns, identifiersHi, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), jsonData); + + } + static final List> updateLastIndexedList = new ArrayList<>(); @Override diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 78e207f989b..066eb267b50 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1601,7 +1601,7 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon searchService.defaultTask().addRunnable(SearchService.PRIORITY.group, () -> { for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) - expMaterial.index(searchService.defaultTask()); + expMaterial.index(searchService.defaultTask(), this); }) ); @@ -1609,7 +1609,7 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon searchService.defaultTask().addRunnable(SearchService.PRIORITY.group, () -> { for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) - expMaterial.index(searchService.defaultTask()); + expMaterial.index(searchService.defaultTask(), this); }) ); }, DbScope.CommitTaskOption.POSTCOMMIT) diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 650e56bc225..06c7ff9b0ae 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -1040,7 +1040,7 @@ private void indexMaterials(final @NotNull SearchService.IndexTask task, final @ List materials = selector.getArrayList(Material.class); materials.forEach(m -> { ExpMaterialImpl expMaterial = new ExpMaterialImpl(m); - var doc = expMaterial.createIndexDocument(); + var doc = expMaterial.createIndexDocument(null); if (doc != null) { task.addResource(doc, SearchService.PRIORITY.bulk); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index b96d08e7677..dcac2532ac9 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -295,7 +295,7 @@ private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.In for (Material m : batch) { ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(task); + impl.index(task, null /* null tableInfo since samples may belong to multiple containers*/); } }); } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index d65975b1c31..b8f3fc8ecca 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -147,6 +147,7 @@ import static org.labkey.api.exp.query.ExpMaterialTable.Column.SampleState; import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.SamplesSchema.SCHEMA_SAMPLES; import static org.labkey.experiment.ExpDataIterators.incrementCounts; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.rollup; @@ -522,14 +523,15 @@ public List> updateRows(User user, Container container, List if (rowId != null) orderedRowIds.add(rowId); } + // Issue 51263: order by RowId to reduce deadlock Collections.sort(orderedRowIds); - // Issue 51263: order by RowId to reduce deadlock + ExpMaterialTableImpl tableInfo = (ExpMaterialTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_SAMPLES).getTable(_sampleType.getName()); ListUtils.partition(orderedRowIds, 100).forEach(sublist -> searchService.defaultTask().addRunnable(SearchService.PRIORITY.group, () -> { for (ExpMaterialImpl expMaterial : ExperimentServiceImpl.get().getExpMaterials(sublist)) - expMaterial.index(searchService.defaultTask()); + expMaterial.index(searchService.defaultTask(), tableInfo); }) ); }, DbScope.CommitTaskOption.POSTCOMMIT); From ed1c790cd37cecadc03ee78498ce7d28c07a23c6 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 29 Apr 2025 12:37:32 -0700 Subject: [PATCH 2/3] fix material index --- .../experiment/api/AbstractRunItemImpl.java | 22 ++++++++++++--- .../labkey/experiment/api/ExpDataImpl.java | 28 +++++-------------- .../experiment/api/ExpMaterialImpl.java | 9 +++--- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java index 5c40a01600c..df3cb58f477 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java @@ -335,13 +335,14 @@ protected HashMap getObjectProperties(TableInfo ti) } public void processIndexValues( + Map props, ExpRunItemTableImpl table, CaseInsensitiveHashSet skipColumns, Set identifiersHi, Set identifiersMed, Set identifiersLo, - Set keywordHi, - Set keywordMed, + Set keywordsHi, + Set keywordsMed, Set keywordsLo, JSONObject jsonData ) @@ -430,11 +431,11 @@ else if (o instanceof Integer) continue; } case keywordsHi -> { - keywordHi.addAll(values); + keywordsHi.addAll(values); continue; } case keywordsMed -> { - keywordMed.addAll(values); + keywordsMed.addAll(values); continue; } case keywordsLo -> { @@ -462,5 +463,18 @@ else if (o instanceof Integer) // ignore } + // === Not stemmed + + props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); + props.put(SearchService.PROPERTY.identifiersMed.toString(), StringUtils.join(identifiersMed, " ")); + props.put(SearchService.PROPERTY.identifiersLo.toString(), StringUtils.join(identifiersLo, " ")); + + + // === Stemmed + + props.put(SearchService.PROPERTY.keywordsHi.toString(), StringUtils.join(keywordsHi, " ")); + props.put(SearchService.PROPERTY.keywordsMed.toString(), StringUtils.join(keywordsMed, " ")); + props.put(SearchService.PROPERTY.keywordsLo.toString(), StringUtils.join(keywordsLo, " ")); + } } diff --git a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java index 5ab764ee6cd..4cc13616158 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java @@ -486,6 +486,7 @@ public void importDataFile(PipelineJob job, XarSource xarSource) throws Experime // Get all text and int strings from the data class for indexing private void getIndexValues( + Map props, ExpDataClassDataTableImpl table, Set identifiersHi, Set identifiersMed, @@ -502,7 +503,7 @@ private void getIndexValues( skipColumns.add("Ancestors"); skipColumns.add("Container"); - processIndexValues(table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); + processIndexValues(props, table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); } @Override @@ -743,12 +744,6 @@ public WebdavResource createDocument(@Nullable ExpDataClassDataTableImpl tableIn tableInfo = (ExpDataClassDataTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), getContainer(), "exp.data").getTable(dc.getName()); } - if (tableInfo != null) - { - // Collect other text columns and lookup display columns - getIndexValues(tableInfo, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); - } - if (null != dc) { ActionURL show = new ActionURL(ExperimentController.ShowDataClassAction.class, getContainer()).addParameter("rowId", dc.getRowId()); @@ -760,22 +755,13 @@ public WebdavResource createDocument(@Nullable ExpDataClassDataTableImpl tableIn body.append(dc.getName()); } - - // === Not stemmed - - props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); - props.put(SearchService.PROPERTY.identifiersMed.toString(), StringUtils.join(identifiersMed, " ")); - props.put(SearchService.PROPERTY.identifiersLo.toString(), StringUtils.join(identifiersLo, " ")); - - - // === Stemmed - - props.put(SearchService.PROPERTY.keywordsHi.toString(), StringUtils.join(keywordsHi, " ")); - props.put(SearchService.PROPERTY.keywordsMed.toString(), StringUtils.join(keywordsMed, " ")); - props.put(SearchService.PROPERTY.keywordsLo.toString(), StringUtils.join(keywordsLo, " ")); + if (tableInfo != null) + { + // Collect other text columns and lookup display columns + getIndexValues(props, tableInfo, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); + } // === Stored, not indexed - if (dc != null && dc.isMedia()) props.put(SearchService.PROPERTY.categories.toString(), expMediaDataCategory.toString()); else diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index 3ec6a1915fe..4c52724223e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -451,16 +451,18 @@ public WebdavResource createIndexDocument(ExpMaterialTableImpl tableInfo) append(body, getSourceApplication()); // Add all String and Integer custom property descriptions and values to body + JSONObject jsonData = new JSONObject(); if (null != getSampleType()) { if (tableInfo == null) tableInfo = (ExpMaterialTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_SAMPLES).getTable(getSampleType().getName()); if (tableInfo != null) - getCustomIndexValues(tableInfo, identifiersHi, new JSONObject()); + getCustomIndexValues(props, tableInfo, jsonData); } props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); + props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); ExpSampleType st = getSampleType(); if (null != st) @@ -514,8 +516,8 @@ public void setLastIndexed(long ms, long modified) // Get all text and int strings from the material properties for indexing private void getCustomIndexValues( + Map props, ExpMaterialTableImpl table, - Set identifiersHi, JSONObject jsonData ) { @@ -523,8 +525,7 @@ private void getCustomIndexValues( for (ExpMaterialTable.Column column : ExpMaterialTable.Column.values()) skipColumns.add(column.name()); - processIndexValues(table, skipColumns, identifiersHi, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), jsonData); - + processIndexValues(props, table, skipColumns, new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), jsonData); } static final List> updateLastIndexedList = new ArrayList<>(); From 2e815084fcadd7cf7a60de53e533a77dd1d9fcd7 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 30 Apr 2025 19:47:45 -0700 Subject: [PATCH 3/3] code review changes --- .../org/labkey/experiment/api/AbstractRunItemImpl.java | 5 +++-- .../src/org/labkey/experiment/api/ExpDataImpl.java | 2 +- .../src/org/labkey/experiment/api/ExpMaterialImpl.java | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java index df3cb58f477..c08d0cb2c38 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.collections.CaseInsensitiveHashSet; @@ -334,9 +335,9 @@ protected HashMap getObjectProperties(TableInfo ti) }); } - public void processIndexValues( + protected void processIndexValues( Map props, - ExpRunItemTableImpl table, + @NotNull ExpRunItemTableImpl table, CaseInsensitiveHashSet skipColumns, Set identifiersHi, Set identifiersMed, diff --git a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java index 4cc13616158..498985c993d 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java @@ -487,7 +487,7 @@ public void importDataFile(PipelineJob job, XarSource xarSource) throws Experime // Get all text and int strings from the data class for indexing private void getIndexValues( Map props, - ExpDataClassDataTableImpl table, + @NotNull ExpDataClassDataTableImpl table, Set identifiersHi, Set identifiersMed, Set identifiersLo, diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index 4c52724223e..d3570ebe4d2 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -412,7 +412,7 @@ public void index(@Nullable SearchService.IndexTask task, @Nullable ExpMaterialT /** returns null if the parent container is no longer available */ @Nullable - public WebdavResource createIndexDocument(ExpMaterialTableImpl tableInfo) + public WebdavResource createIndexDocument(@Nullable ExpMaterialTableImpl tableInfo) { Container container = getContainer(); if (container == null) @@ -458,10 +458,9 @@ public WebdavResource createIndexDocument(ExpMaterialTableImpl tableInfo) tableInfo = (ExpMaterialTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_SAMPLES).getTable(getSampleType().getName()); if (tableInfo != null) - getCustomIndexValues(props, tableInfo, jsonData); + getCustomIndexValues(props, tableInfo, identifiersHi, jsonData); } - props.put(SearchService.PROPERTY.identifiersHi.toString(), StringUtils.join(identifiersHi, " ")); props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); ExpSampleType st = getSampleType(); @@ -517,7 +516,8 @@ public void setLastIndexed(long ms, long modified) // Get all text and int strings from the material properties for indexing private void getCustomIndexValues( Map props, - ExpMaterialTableImpl table, + @NotNull ExpMaterialTableImpl table, + Set identifiersHi, JSONObject jsonData ) { @@ -525,7 +525,7 @@ private void getCustomIndexValues( for (ExpMaterialTable.Column column : ExpMaterialTable.Column.values()) skipColumns.add(column.name()); - processIndexValues(props, table, skipColumns, new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), jsonData); + processIndexValues(props, table, skipColumns, identifiersHi, new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>(), jsonData); } static final List> updateLastIndexedList = new ArrayList<>();