From 2fc9a0257c0af59d1897f192a7c93033f4846b0c Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 25 Nov 2025 21:34:43 -0800 Subject: [PATCH 01/18] Support additional unit types --- api/src/org/labkey/api/ontology/Quantity.java | 4 + api/src/org/labkey/api/ontology/Unit.java | 34 + .../experiment/api/SampleTypeServiceImpl.java | 4758 +++++++++-------- .../api/SampleTypeUpdateServiceDI.java | 8 +- 4 files changed, 2433 insertions(+), 2371 deletions(-) diff --git a/api/src/org/labkey/api/ontology/Quantity.java b/api/src/org/labkey/api/ontology/Quantity.java index ccbb26db1d3..aa4c54ea363 100644 --- a/api/src/org/labkey/api/ontology/Quantity.java +++ b/api/src/org/labkey/api/ontology/Quantity.java @@ -465,6 +465,10 @@ public void testParse() assertEquals(Quantity.of(0, Unit.count), parse("0 units")); assertEquals(Quantity.of(0, Unit.count), parse("0count")); + assertEquals(Quantity.of(1, Unit.count), parse("1", Unit.box)); + assertEquals(Quantity.of(1, Unit.unit), parse("1", Unit.blocks)); + assertEquals(Quantity.of(1, Unit.cells), parse("1", Unit.tests)); + assertEquals(parse("1000mg", Unit.g), parse("0.001kg", Unit.g)); assertEquals(parse(" 1000mg", Unit.g), parse("0.001kg", Unit.g)); assertEquals(parse("1000mg ", Unit.g), parse("0.001kg", Unit.g)); diff --git a/api/src/org/labkey/api/ontology/Unit.java b/api/src/org/labkey/api/ontology/Unit.java index 1cb963687a7..7e281b1285d 100644 --- a/api/src/org/labkey/api/ontology/Unit.java +++ b/api/src/org/labkey/api/ontology/Unit.java @@ -19,6 +19,33 @@ public enum Unit count(KindOfQuantity.Count, unit, 1.0, 2, "count", Quantity.class, "count", "count"), + pcs(KindOfQuantity.Count, unit, 1.0, 2, "pcs", + Quantity.class, + "pcs", "pcs"), + pack(KindOfQuantity.Count, unit, 1.0, 2, "pack", + Quantity.class, + "pack", "packs"), + blocks(KindOfQuantity.Count, unit, 1.0, 2, "blocks", + Quantity.class, + "block", "blocks"), + slides(KindOfQuantity.Count, unit, 1.0, 2, "slides", + Quantity.class, + "slide", "slides"), + cells(KindOfQuantity.Count, unit, 1.0, 2, "cells", + Quantity.class, + "cell", "cells"), + box(KindOfQuantity.Count, unit, 1.0, 2, "box", + Quantity.class, + "box", "boxes"), + kit(KindOfQuantity.Count, unit, 1.0, 2, "kit", + Quantity.class, + "kit", "kits"), + tests(KindOfQuantity.Count, unit, 1.0, 2, "tests", + Quantity.class, + "test", "tests"), + bottle(KindOfQuantity.Count, unit, 1.0, 2, "bottle", + Quantity.class, + "bottle", "bottles"), mL(KindOfQuantity.Volume, null, 1e0, 6, "mL", Quantity.Volume_ml.class, @@ -206,6 +233,7 @@ public void testIsBase() assertFalse(Unit.kg.isBase()); assertTrue(Unit.unit.isBase()); assertFalse(Unit.count.isBase()); + assertFalse(Unit.bottle.isBase()); } @Test @@ -217,6 +245,12 @@ public void testIsCompatible() assertTrue(Unit.g.isCompatible(Unit.mg)); assertFalse(Unit.g.isCompatible(Unit.mL)); assertTrue(Unit.unit.isCompatible(Unit.count)); + assertTrue(Unit.unit.isCompatible(Unit.pcs)); + assertTrue(Unit.unit.isCompatible(Unit.pack)); + assertTrue(Unit.unit.isCompatible(Unit.bottle)); + assertTrue(Unit.unit.isCompatible(Unit.blocks)); + assertTrue(Unit.unit.isCompatible(Unit.box)); + assertTrue(Unit.unit.isCompatible(Unit.slides)); assertFalse(Unit.unit.isCompatible(Unit.mL)); assertFalse(Unit.mL.isCompatible(null)); } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 013bf5d01ac..b55a0a20897 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2370 +1,2388 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container, false); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container, - boolean useRootMaterialLSID - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates, useRootMaterialLSID); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append(useRootMaterialLSID ? "a.rootMaterialLsid = m.lsid" : "a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql; - if (useRootMaterialLSID) - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialLSID as rootLsid, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialLSID <> LSID AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialLSID - ) AS c ON m.lsid = c.rootLsid - WHERE m.rootmateriallsid = m.LSID AND m.rowid\s"""); - } - else - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - } - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append(useRootMaterialLSID ? "parent.lsid = aliquot.rootmateriallsid" : "parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append(useRootMaterialLSID ? "aliquot.rootmateriallsid <> aliquot.lsid" : "aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + + SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") + .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) + .append("FROM exp.material \n") + .append("WHERE rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND rowid != rootmaterialrowid\n") + .append(" GROUP BY rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index a9691bc3f55..d87c8ab218c 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -87,6 +87,7 @@ import org.labkey.api.exp.query.SamplesSchema; import org.labkey.api.gwt.client.AuditBehaviorType; import org.labkey.api.inventory.InventoryService; +import org.labkey.api.ontology.KindOfQuantity; import org.labkey.api.ontology.Quantity; import org.labkey.api.ontology.Unit; import org.labkey.api.qc.DataState; @@ -2074,8 +2075,13 @@ public static Object getValue(Object o, Object amountObj, boolean haveAmountCol, Unit validatedUnit = SampleTypeService.get().getValidatedUnit(o, baseUnit, sampleTypeName); + if (validatedUnit != null && baseUnit != null && KindOfQuantity.Count == validatedUnit.getKindOfQuantity() && validatedUnit.getValue() == baseUnit.getValue()) + { + // if both units are 'count' units and have the same value, prefer returning provided unit name + return validatedUnit.name(); + } // if there's a base unit, return the base unit name otherwise return the name of the given unit - return validatedUnit == null ? null : baseUnit != null ? baseUnit.name() : validatedUnit.name(); + return validatedUnit == null ? null : baseUnit != null ? baseUnit.name() : validatedUnit.name(); // prefer provided count } @Override From fee03aff694bc19d57a2d98baf7c72fe38738b6f Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 26 Nov 2025 10:03:24 -0800 Subject: [PATCH 02/18] fix test --- api/src/org/labkey/api/ontology/KindOfQuantity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java index cd51aeb5e6b..8b6f20fca8d 100644 --- a/api/src/org/labkey/api/ontology/KindOfQuantity.java +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -39,7 +39,7 @@ public List getCommonUnits() @Override public List getCommonUnits() { - return List.of(Unit.unit); + return List.of(Unit.unit, Unit.pcs, Unit.pack, Unit.blocks, Unit.slides, Unit.cells, Unit.box, Unit.kit, Unit.tests, Unit.bottle); } }; From 9b41448ced577585e19d87d2ef33781a49347740 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 26 Nov 2025 14:35:05 -0800 Subject: [PATCH 03/18] jest integration tests --- .../labkey/api/ontology/KindOfQuantity.java | 2 +- .../test/integration/SampleTypeCrud.ispec.ts | 241 ++++++++++++++++++ .../experiment/api/SampleTypeServiceImpl.java | 10 +- 3 files changed, 251 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java index 8b6f20fca8d..aac61fc5f92 100644 --- a/api/src/org/labkey/api/ontology/KindOfQuantity.java +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -29,7 +29,7 @@ public List getCommonUnits() @Override public List getCommonUnits() { - return List.of(Unit.kg, Unit.g, Unit.mg); + return List.of(Unit.kg, Unit.g, Unit.mg, Unit.ug, Unit.ng, Unit.pg); } }, diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index a600c34d2b4..84a02d9eee1 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -974,6 +974,12 @@ describe('Amount/Unit CRUD', () => { expect(errorMsg.text).toContain(NO_AMOUNT_ERROR); errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tL", dataType, "INSERT", topFolderOptions, editorUserOptions); expect(errorMsg.text).toContain(INCOMPATIBLE_ERROR); + errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tunit", dataType, "INSERT", topFolderOptions, editorUserOptions); + expect(errorMsg.text).toContain('Units value (unit) is not compatible with the ' + dataType + ' display units (g).'); + errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tcells", dataType, "INSERT", topFolderOptions, editorUserOptions); + expect(errorMsg.text).toContain('Units value (cells) is not compatible with the ' + dataType + ' display units (g).'); + errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tbogus", dataType, "INSERT", topFolderOptions, editorUserOptions); + expect(errorMsg.text).toContain('Unsupported Units value (bogus). Supported values are: kg, g, mg, ug, ng, pg.'); errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t-1.1\tkg", dataType, "INSERT", topFolderOptions, editorUserOptions); expect(errorMsg.text).toContain(NEGATIVE_ERROR); errorMsg = await ExperimentCRUDUtils.importCrossTypeData(server, "Name\tStoredAmount\tUnits\tSampleType\nData1\t-1.1\tkg\t" + dataType ,'IMPORT', topFolderOptions, adminOptions, true); @@ -1085,5 +1091,240 @@ describe('Amount/Unit CRUD', () => { }); + it ("Test units conversion on insert/update", async () => { + const sampleTypeMass = 'SampleTypeWithMassUnits'; + const sampleTypeVolume = 'SampleTypeWithVolumeUnits'; + const sampleTypeCount = 'SampleTypeWithCountUnits'; + + const sampleTypeUnits = { + [sampleTypeMass]: 'ug', + [sampleTypeVolume]: 'L', + [sampleTypeCount]: 'unit' + }; + + for (const [dataType, unit] of Object.entries(sampleTypeUnits)) { + const createPayload = { + kind: 'SampleSet', + domainDesign: { name: dataType, fields: [{ name: 'Name' }] }, + options: { + name: dataType, + metricUnit: unit + } + }; + await server.post('property', 'createDomain', createPayload, {...topFolderOptions, ...designerReaderOptions}).expect(successfulResponse); + } + + let sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, [ + {name: 'S-pg', amount: 4.56, units: 'pg'}, + {name: 'S-ng', amount: 4.56, units: 'ng'}, + {name: 'S-ug', amount: 4.56, units: 'ug'}, + {name: 'S-mg', amount: 4.56, units: 'mg'}, + {name: 'S-g', amount: 4.56, units: 'g'}, + {name: 'S-kg', amount: 4.56, units: 'kg'}, + ], 'samples', sampleTypeMass, topFolderOptions, editorUserOptions); + + // check for storedamount in g + let expectedRawAmounts : {} = { + 'S-pg': 4.56e-12, + 'S-ng': 4.56e-9, + 'S-ug': 4.56e-6, + 'S-mg': 0.00456, + 'S-g': 4.56, + 'S-kg': 4560, + }; + let expectedStoredAmounts : {} = { + 'S-pg': 4.56e-6, + 'S-ng': 4.56e-3, + 'S-ug': 4.56, + 'S-mg': 4560, + 'S-g': 4.56e6, + 'S-kg': 4.56e9, + }; + + for (const sampleRow of sampleRowsWithUnits) { + const sampleName = caseInsensitive(sampleRow, 'name'); + let sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeMass, 'StoredAmount,Units,RawAmount,RawUnits', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawAmount')).toBeCloseTo(expectedRawAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'StoredAmount')).toBeCloseTo(expectedStoredAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual('g'); + expect(caseInsensitive(sampleData, 'Units')).toEqual('ug'); + await server.post('query', 'updateRows', { + schemaName: 'samples', + queryName: sampleTypeMass, + rows: [{ + amount: 6.54, + units: sampleName.substring(2), + rowId: caseInsensitive(sampleRow, 'rowId') + }] + }, { ...topFolderOptions, ...editorUserOptions }).expect(successfulResponse); + } + + expectedRawAmounts = { + 'S-pg': 6.54e-12, + 'S-ng': 6.54e-9, + 'S-ug': 6.54e-6, + 'S-mg': 0.00654, + 'S-g': 6.54, + 'S-kg': 6540, + }; + expectedStoredAmounts = { + 'S-pg': 6.54e-6, + 'S-ng': 6.54e-3, + 'S-ug': 6.54, + 'S-mg': 6540, + 'S-g': 6.54e6, + 'S-kg': 6.54e9, + }; + for (const sampleRow of sampleRowsWithUnits) { + const sampleName = caseInsensitive(sampleRow, 'name'); + let sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeMass, 'StoredAmount,Units,RawAmount,RawUnits', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawAmount')).toBeCloseTo(expectedRawAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'StoredAmount')).toBeCloseTo(expectedStoredAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual('g'); + expect(caseInsensitive(sampleData, 'Units')).toEqual('ug'); + } + + sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, [ + {name: 'S-L', amount: 4.56, units: 'L'}, + {name: 'S-mL', amount: 4.56, units: 'mL'}, + {name: 'S-uL', amount: 4.56, units: 'uL'}, + ], 'samples', sampleTypeVolume, topFolderOptions, editorUserOptions); + + // check for storedamount in mL + expectedRawAmounts = { + 'S-L': 4560, + 'S-mL': 4.56, + 'S-uL': 0.00456, + }; + // stored amount is in L + expectedStoredAmounts = { + 'S-L': 4.56, + 'S-mL': 0.00456, + 'S-uL': 4.56e-6, + } + for (const sampleRow of sampleRowsWithUnits) { + const sampleName = caseInsensitive(sampleRow, 'name'); + const sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeVolume, 'StoredAmount,Units,RawAmount,RawUnits', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawAmount')).toBeCloseTo(expectedRawAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'StoredAmount')).toBeCloseTo(expectedStoredAmounts[sampleName]); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual('mL'); + expect(caseInsensitive(sampleData, 'Units')).toEqual('L'); + } + + const countRows = [ + {name: 'S-unit', amount: 4.56, units: 'unit'}, + {name: 'S-pcs', amount: 4.56, units: 'pcs'}, + {name: 'S-kit', amount: 4.56, units: 'kit'}, + {name: 'S-cells', amount: 4.56, units: 'cells'} + ] + sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, countRows, 'samples', sampleTypeCount, topFolderOptions, editorUserOptions); + + for (const sampleRow of sampleRowsWithUnits) { + const sampleName = caseInsensitive(sampleRow, 'name'); + const usedUnit = sampleName.substring(2); + const sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeCount, 'StoredAmount,Units,RawAmount,RawUnits', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawAmount')).toBeCloseTo(4.56); + expect(caseInsensitive(sampleData, 'StoredAmount')).toBeCloseTo(4.56); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual(usedUnit); + expect(caseInsensitive(sampleData, 'Units')).toEqual(usedUnit); + + await server.post('query', 'updateRows', { + schemaName: 'samples', + queryName: sampleTypeCount, + rows: [{ + amount: 6.54, + units: usedUnit, + rowId: caseInsensitive(sampleRow, 'rowId') + }] + }, { ...topFolderOptions, ...editorUserOptions }).expect(successfulResponse); + } + + for (const sampleRow of sampleRowsWithUnits) { + const sampleName = caseInsensitive(sampleRow, 'name'); + const usedUnit = sampleName.substring(2); + const sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeCount, 'StoredAmount,Units,RawAmount,RawUnits', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawAmount')).toBeCloseTo(6.54); + expect(caseInsensitive(sampleData, 'StoredAmount')).toBeCloseTo(6.54); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual(usedUnit); + expect(caseInsensitive(sampleData, 'Units')).toEqual(usedUnit); + } + + }) + + async function verifyCountTypeAliquotRollup(sampleTypeName: string, hasSampleTypeDisplayUnit: boolean) { + const dataRows = [ + {name: 'S-no-amount'}, + {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs1', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs2', amount: 2, units: 'pcs'}, + {name: 'S-unit', amount: 1, units: 'unit'}, + {AliquotedFrom: 'S-unit', name: 'S-unit-unit1', amount: 2, units: 'unit'}, + {AliquotedFrom: 'S-unit', name: 'S-unit-unit2', amount: 2, units: 'unit'}, + {name: 'S-pcs', amount: 1, units: 'pcs'}, + {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs1', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs2', amount: 2, units: 'pcs'}, + {name: 'S-kit', amount: 1, units: 'kit'}, + {AliquotedFrom: 'S-kit', name: 'S-kit-pcs1', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-kit', name: 'S-kit-pcs2', amount: 2, units: 'pcs'}, + {name: 'S-cells', amount: 1, units: 'cells'}, + {AliquotedFrom: 'S-cells', name: 'S-cells-pcs1', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-cells', name: 'S-cells-cells2', amount: 2, units: 'cells'}, + ] + + const insertedResults = await ExperimentCRUDUtils.insertRows(server, dataRows, 'samples', sampleTypeName, topFolderOptions, editorUserOptions); + const insertedMap = {}; + for (const row of insertedResults) { + insertedMap[caseInsensitive(row, 'name')] = row; + } + + let expectedAliquotUnit = { + 'S-no-amount': 'pcs', + 'S-unit': 'unit', + 'S-pcs': 'pcs', + 'S-kit': 'pcs', + 'S-cells': hasSampleTypeDisplayUnit ? 'unit' : 'cells', + }; + + // for each expectedRollupAmounts + for (const [sampleName, expectedAliquotUnitValue] of Object.entries(expectedAliquotUnit)) { + let parentUnit = sampleName.substring(2); + if (parentUnit === 'no-amount') { + parentUnit = null; + } + const sampleData = await ExperimentCRUDUtils.getSampleDataByName(server, sampleName, sampleTypeName, 'Units,RawUnits,AliquotVolume,AliquotCount,AliquotUnit', topFolderOptions, readerUserOptions); + expect(caseInsensitive(sampleData, 'RawUnits')).toEqual(parentUnit); + expect(caseInsensitive(sampleData, 'Units')).toEqual(parentUnit); + expect(caseInsensitive(sampleData, 'AliquotVolume')).toEqual(4); + expect(caseInsensitive(sampleData, 'AliquotCount')).toEqual(2); + expect(caseInsensitive(sampleData, 'AliquotUnit')).toEqual(expectedAliquotUnitValue); + + } + } + + it ("Test aliquot rollup for count display unit", async () => { + let dataType = 'SampleTypeAliquotWithCountUnit'; + let createPayload : {} = { + kind: 'SampleSet', + domainDesign: { name: dataType, fields: [{ name: 'Name' }] }, + options: { + name: dataType, + metricUnit: 'unit' + } + }; + await server.post('property', 'createDomain', createPayload, {...topFolderOptions, ...designerReaderOptions}).expect(successfulResponse); + await verifyCountTypeAliquotRollup(dataType, true); + + dataType = 'SampleTypeAliquoNoDisplayUnit'; + createPayload = { + kind: 'SampleSet', + domainDesign: { name: dataType, fields: [{ name: 'Name' }] }, + options: { + name: dataType, + } + }; + await server.post('property', 'createDomain', createPayload, {...topFolderOptions, ...designerReaderOptions}).expect(successfulResponse); + await verifyCountTypeAliquotRollup(dataType, false); + + }) + }); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index b55a0a20897..e99bdae6fc3 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -253,7 +253,9 @@ else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) List commonUnits = getSupportedUnits(); if (mUnit == null || !commonUnits.contains(mUnit)) { - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); } if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); @@ -1639,10 +1641,16 @@ private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + Unit totalUnit = null; String totalUnitsStr; if (!StringUtils.isEmpty(sampleTypeUnitsStr)) totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; else if (!StringUtils.isEmpty(sampleItemUnitsStr)) totalUnitsStr = sampleItemUnitsStr; else // use the unit of the first aliquot if there are no other indications From 4bd66f9d3045d824f5dff6bc8eacd80deae0b5ac Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 26 Nov 2025 14:39:05 -0800 Subject: [PATCH 04/18] crlf --- .../experiment/api/SampleTypeServiceImpl.java | 4792 ++++++++--------- 1 file changed, 2396 insertions(+), 2396 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index e99bdae6fc3..b678b6aab64 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2396 +1,2396 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - if (defaultUnits != null) - commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - if (!StringUtils.isEmpty(sampleTypeUnit)) - { - // if sample type has unit, use it for simple rollup without need for conversion - Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); - String baseUnit = sampleTypeBaseUnit.name(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> - { - if (sublist.isEmpty()) - return; - - int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); - - SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") - .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") - .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) - .append("FROM exp.material \n") - .append("WHERE rootmaterialrowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()) - .append(" AND rowid != rootmaterialrowid\n") - .append(" GROUP BY rootmaterialrowid\n"); - - SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) - .append("FROM (") - .append(statsSql) - .append(") AS stats\n") - .append("WHERE m.rowid = stats.rootmaterialrowid" - ); - new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); - - // Now clear out rollups for samples that have zero aliquots - SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = 0, availablealiquotvolume = 0, ") - .append("aliquotunit = ?\n").add(baseUnit) - .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()); - new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); - - }); - } - else - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); - boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit - totalUnitsStr = volumeUnits.get(0).unit; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append("a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append("parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append("aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - service.getValidatedUnit("box", Unit.unit, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + + SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") + .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) + .append("FROM exp.material \n") + .append("WHERE rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND rowid != rootmaterialrowid\n") + .append(" GROUP BY rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} From 0f5d249aa3bdb6b88471c7044bf909e8962be4f8 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 26 Nov 2025 19:49:25 -0800 Subject: [PATCH 05/18] Support additional unit types --- .../labkey/api/ontology/KindOfQuantity.java | 2 +- .../experiment/api/SampleTypeServiceImpl.java | 4831 +++++++++-------- .../api/SampleTypeUpdateServiceDI.java | 2 +- 3 files changed, 2437 insertions(+), 2398 deletions(-) diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java index aac61fc5f92..4c7f3e2568e 100644 --- a/api/src/org/labkey/api/ontology/KindOfQuantity.java +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -39,7 +39,7 @@ public List getCommonUnits() @Override public List getCommonUnits() { - return List.of(Unit.unit, Unit.pcs, Unit.pack, Unit.blocks, Unit.slides, Unit.cells, Unit.box, Unit.kit, Unit.tests, Unit.bottle); + return List.of(Unit.unit, Unit.blocks, Unit.bottle, Unit.box, Unit.cells, Unit.kit, Unit.pack, Unit.pcs, Unit.slides, Unit.tests); } }; diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index b678b6aab64..b3447939d83 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2396 +1,2435 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - if (defaultUnits != null) - commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - if (!StringUtils.isEmpty(sampleTypeUnit)) - { - // if sample type has unit, use it for simple rollup without need for conversion - Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); - String baseUnit = sampleTypeBaseUnit.name(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> - { - if (sublist.isEmpty()) - return; - - int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); - - SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") - .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") - .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) - .append("FROM exp.material \n") - .append("WHERE rootmaterialrowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()) - .append(" AND rowid != rootmaterialrowid\n") - .append(" GROUP BY rootmaterialrowid\n"); - - SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) - .append("FROM (") - .append(statsSql) - .append(") AS stats\n") - .append("WHERE m.rowid = stats.rootmaterialrowid" - ); - new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); - - // Now clear out rollups for samples that have zero aliquots - SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = 0, availablealiquotvolume = 0, ") - .append("aliquotunit = ?\n").add(baseUnit) - .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()); - new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); - - }); - } - else - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); - boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit - totalUnitsStr = volumeUnits.get(0).unit; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append("a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append("parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append("aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - service.getValidatedUnit("box", Unit.unit, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) + { + if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) + return false; + + boolean hasToValidateUnit = true; + Unit newUnit = Unit.fromName(newUnitStr); + if (!StringUtils.isEmpty(st.getMetricUnit())) + { + Unit oldUnit = Unit.fromName(st.getMetricUnit()); + if (oldUnit != null && newUnit != null) + hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); + } + + if (hasToValidateUnit) + { + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); + filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); + if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) + { + List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); + filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); + } + else + filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); + + TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); + return ts.exists(); + } + + return false; + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + + if (hasIncompatibleUnits(st, options.getMetricUnit())) + throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); + + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + + SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") + .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) + .append("FROM exp.material \n") + .append("WHERE rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND rowid != rootmaterialrowid\n") + .append(" GROUP BY rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index d87c8ab218c..d9e5a25995c 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -2081,7 +2081,7 @@ public static Object getValue(Object o, Object amountObj, boolean haveAmountCol, return validatedUnit.name(); } // if there's a base unit, return the base unit name otherwise return the name of the given unit - return validatedUnit == null ? null : baseUnit != null ? baseUnit.name() : validatedUnit.name(); // prefer provided count + return validatedUnit == null ? null : baseUnit != null ? baseUnit.name() : validatedUnit.name(); } @Override From a46c8eccd7e5dbbb2cff9202d3b41b3291769dd5 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 26 Nov 2025 19:50:51 -0800 Subject: [PATCH 06/18] crlf --- .../experiment/api/SampleTypeServiceImpl.java | 4870 ++++++++--------- 1 file changed, 2435 insertions(+), 2435 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index b3447939d83..32d83c215a1 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2435 +1,2435 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - if (defaultUnits != null) - commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) - { - if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) - return false; - - boolean hasToValidateUnit = true; - Unit newUnit = Unit.fromName(newUnitStr); - if (!StringUtils.isEmpty(st.getMetricUnit())) - { - Unit oldUnit = Unit.fromName(st.getMetricUnit()); - if (oldUnit != null && newUnit != null) - hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); - } - - if (hasToValidateUnit) - { - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); - filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); - if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) - { - List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); - filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); - } - else - filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); - - TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); - return ts.exists(); - } - - return false; - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - - if (hasIncompatibleUnits(st, options.getMetricUnit())) - throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); - - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - if (!StringUtils.isEmpty(sampleTypeUnit)) - { - // if sample type has unit, use it for simple rollup without need for conversion - Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); - String baseUnit = sampleTypeBaseUnit.name(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> - { - if (sublist.isEmpty()) - return; - - int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); - - SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") - .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") - .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) - .append("FROM exp.material \n") - .append("WHERE rootmaterialrowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()) - .append(" AND rowid != rootmaterialrowid\n") - .append(" GROUP BY rootmaterialrowid\n"); - - SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) - .append("FROM (") - .append(statsSql) - .append(") AS stats\n") - .append("WHERE m.rowid = stats.rootmaterialrowid" - ); - new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); - - // Now clear out rollups for samples that have zero aliquots - SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = 0, availablealiquotvolume = 0, ") - .append("aliquotunit = ?\n").add(baseUnit) - .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()); - new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); - - }); - } - else - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); - boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit - totalUnitsStr = volumeUnits.get(0).unit; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append("a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append("parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append("aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - service.getValidatedUnit("box", Unit.unit, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) + { + if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) + return false; + + boolean hasToValidateUnit = true; + Unit newUnit = Unit.fromName(newUnitStr); + if (!StringUtils.isEmpty(st.getMetricUnit())) + { + Unit oldUnit = Unit.fromName(st.getMetricUnit()); + if (oldUnit != null && newUnit != null) + hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); + } + + if (hasToValidateUnit) + { + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); + filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); + if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) + { + List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); + filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); + } + else + filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); + + TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); + return ts.exists(); + } + + return false; + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + + if (hasIncompatibleUnits(st, options.getMetricUnit())) + throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); + + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + + SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") + .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) + .append("FROM exp.material \n") + .append("WHERE rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND rowid != rootmaterialrowid\n") + .append(" GROUP BY rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} From e8967815b7c36bfb60992e828632e47b6dca5407 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 28 Nov 2025 18:17:43 -0800 Subject: [PATCH 07/18] Aliquot amount in display unit --- .../api/exp/query/ExpMaterialTable.java | 3 + api/src/org/labkey/api/ontology/Unit.java | 12 +- .../experiment/api/ExpMaterialTableImpl.java | 146 +- .../experiment/api/SampleTypeServiceImpl.java | 4901 +++++++++-------- 4 files changed, 2615 insertions(+), 2447 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java index 6c856381460..82b19f98578 100644 --- a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java +++ b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java @@ -52,6 +52,9 @@ enum Column Properties, Property, QueryableInputs, + RawAliquotUnit, + RawAliquotVolume, + RawAvailableAliquotVolume, RawAmount(true), RawUnits, RootMaterialRowId, diff --git a/api/src/org/labkey/api/ontology/Unit.java b/api/src/org/labkey/api/ontology/Unit.java index 7e281b1285d..ade6e995bd6 100644 --- a/api/src/org/labkey/api/ontology/Unit.java +++ b/api/src/org/labkey/api/ontology/Unit.java @@ -78,27 +78,27 @@ public enum Unit "picoliter", "picoliters", "pl", "picolitre", "picolitres"), - g(KindOfQuantity.Mass, null, 1e0, 9, "g", + g(KindOfQuantity.Mass, null, 1e0, 12, "g", Quantity.Mass_g.class, "gram", "grams"), - Mg(KindOfQuantity.Mass, g, 1e6, 12, "Mg", + Mg(KindOfQuantity.Mass, g, 1e6, 15, "Mg", Quantity.Mass_Megag.class, "megagram", "megagrams", "tonne", "tonnes"), - kg(KindOfQuantity.Mass, g, 1e3, 12, "kg", + kg(KindOfQuantity.Mass, g, 1e3, 15, "kg", Quantity.Mass_kg.class, "kilogram", "kilograms"), - mg(KindOfQuantity.Mass, g, 1e-3, 6, "mg", + mg(KindOfQuantity.Mass, g, 1e-3, 9, "mg", Quantity.Mass_mg.class, "milligram", "milligrams"), - ug(KindOfQuantity.Mass, g, 1e-6, 3, "ug", + ug(KindOfQuantity.Mass, g, 1e-6, 6, "ug", Quantity.Mass_ug.class, "microgram", "micrograms", "μg"), ng(KindOfQuantity.Mass, g, 1e-9, 3, "ng", Quantity.Mass_ng.class, "nanogram", "nanograms"), - pg(KindOfQuantity.Mass, g, 1e-12, 3, "pg", + pg(KindOfQuantity.Mass, g, 1e-12, 0, "pg", Quantity.Mass_pg.class, "picogram", "picograms"); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 3843fc1b6c0..ce36d961059 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -566,30 +566,93 @@ public StringExpression getURL(ColumnInfo parent) ret.setLabel(ALIQUOT_COUNT_LABEL); return ret; } - case AliquotVolume -> + case RawAliquotVolume -> { var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); - ret.setLabel(ALIQUOT_VOLUME_LABEL); + ret.setLabel("Raw " + ALIQUOT_VOLUME_LABEL); + ret.setShownInDetailsView(false); return ret; } - case AvailableAliquotVolume -> + case AliquotVolume -> + { + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AliquotVolume.name(), Column.AliquotUnit.name(), ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); + columnInfo.setDescription("The total amount of this sample's aliquots, in the display unit for the sample type, currently on hand."); + return columnInfo; + } + else + { + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); + ret.setLabel(ALIQUOT_VOLUME_LABEL); + return ret; + } + } + case RawAvailableAliquotVolume -> { var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); - ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); + ret.setLabel("Raw " + AVAILABLE_ALIQUOT_VOLUME_LABEL); + ret.setShownInDetailsView(false); return ret; } + case AvailableAliquotVolume -> + { + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AvailableAliquotVolume.name(), Column.AliquotUnit.name(), AVAILABLE_ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); + columnInfo.setDescription("The total amount of this sample's aliquots that's available, in the display unit for the sample type, currently on hand."); + return columnInfo; + } + else + { + var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); + ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); + return ret; + } + } case AvailableAliquotCount -> { var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotCount.name())); ret.setLabel(AVAILABLE_ALIQUOT_COUNT_LABEL); return ret; } - case AliquotUnit -> + case RawAliquotUnit -> { var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); ret.setShownInDetailsView(false); + ret.setLabel("Raw Aliquot Unit"); return ret; } + case AliquotUnit -> + { + ForeignKey fk = new LookupForeignKey("Value", "Value") + { + @Override + public @Nullable TableInfo getLookupTableInfo() + { + return getExpSchema().getTable(ExpSchema.MEASUREMENT_UNITS_TABLE); + } + }; + + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.AliquotUnit.name(), typeUnit); + columnInfo.setFk(fk); + columnInfo.setDescription("The sample type display units associated with the AliquotAmount for this sample."); + return columnInfo; + } + else + { + var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); + ret.setShownInDetailsView(false); + return ret; + } + } case MaterialExpDate -> { var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); @@ -881,6 +944,75 @@ public void addQueryFieldKeys(Set keys) rawUnitsColumn.setShownInInsertView(false); rawUnitsColumn.setShownInUpdateView(false); + var rawAliquotVolumeColumn = addColumn(Column.RawAliquotVolume); + rawAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(AliquotVolume)); + + } + }; + } + }); + rawAliquotVolumeColumn.setHidden(true); + rawAliquotVolumeColumn.setShownInDetailsView(false); + rawAliquotVolumeColumn.setShownInInsertView(false); + rawAliquotVolumeColumn.setShownInUpdateView(false); + + var rawAvailableAliquotVolumeColumn = addColumn(Column.RawAvailableAliquotVolume); + rawAvailableAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(AvailableAliquotVolume)); + + } + }; + } + }); + rawAvailableAliquotVolumeColumn.setHidden(true); + rawAvailableAliquotVolumeColumn.setShownInDetailsView(false); + rawAvailableAliquotVolumeColumn.setShownInInsertView(false); + rawAvailableAliquotVolumeColumn.setShownInUpdateView(false); + + var rawAliquotUnitColumn = addColumn(Column.RawAliquotUnit); + rawAliquotUnitColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(Column.AliquotUnit)); + + } + }; + } + }); + rawAliquotUnitColumn.setHidden(true); + rawAliquotUnitColumn.setShownInDetailsView(false); + rawAliquotUnitColumn.setShownInInsertView(false); + rawAliquotUnitColumn.setShownInUpdateView(false); + if (InventoryService.get() != null && (st == null || !st.isMedia())) defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); @@ -1132,6 +1264,8 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); + if (selectedColumns.contains(new FieldKey(null, AliquotVolume.name())) || selectedColumns.contains(new FieldKey(null, AvailableAliquotVolume.name()))) + selectedColumns.add(new FieldKey(null, Column.AliquotUnit.name())); selectedColumns.addAll(wrappedFieldKeys); if (null != getFilter()) selectedColumns.addAll(getFilter().getAllFieldKeys()); @@ -1616,7 +1750,7 @@ private static class SampleTypeUnitDisplayColumn extends ExprColumn { public SampleTypeUnitDisplayColumn(TableInfo parent, String unitFieldName, Unit typeUnit) { - super(parent, FieldKey.fromParts(Column.Units.name()), new SQLFragment( + super(parent, FieldKey.fromParts(unitFieldName), new SQLFragment( "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) .append(" = ? THEN ? ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) .append(" END)") diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 32d83c215a1..e3434a9ac86 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2435 +1,2466 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - if (defaultUnits != null) - commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) - { - if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) - return false; - - boolean hasToValidateUnit = true; - Unit newUnit = Unit.fromName(newUnitStr); - if (!StringUtils.isEmpty(st.getMetricUnit())) - { - Unit oldUnit = Unit.fromName(st.getMetricUnit()); - if (oldUnit != null && newUnit != null) - hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); - } - - if (hasToValidateUnit) - { - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); - filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); - if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) - { - List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); - filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); - } - else - filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); - - TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); - return ts.exists(); - } - - return false; - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - - if (hasIncompatibleUnits(st, options.getMetricUnit())) - throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); - - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - if (!StringUtils.isEmpty(sampleTypeUnit)) - { - // if sample type has unit, use it for simple rollup without need for conversion - Unit sampleTypeBaseUnit = Unit.valueOf(sampleTypeUnit).getBase(); - String baseUnit = sampleTypeBaseUnit.name(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> - { - if (sublist.isEmpty()) - return; - - int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); - - SQLFragment statsSql = new SQLFragment("SELECT rootmaterialrowid, SUM(storedamount) AS total_volume, \n") - .append("SUM(CASE WHEN samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN storedamount ELSE 0 END) AS avail_volume, \n") - .append("CASE WHEN MIN(units) = MAX(units) THEN MIN(units) ELSE ? END AS common_unit \n").add(sampleTypeUnit) - .append("FROM exp.material \n") - .append("WHERE rootmaterialrowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()) - .append(" AND rowid != rootmaterialrowid\n") - .append(" GROUP BY rootmaterialrowid\n"); - - SQLFragment quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) - .append("FROM (") - .append(statsSql) - .append(") AS stats\n") - .append("WHERE m.rowid = stats.rootmaterialrowid" - ); - new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); - - // Now clear out rollups for samples that have zero aliquots - SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = 0, availablealiquotvolume = 0, ") - .append("aliquotunit = ?\n").add(baseUnit) - .append("WHERE m.rowid = m.rootmaterialrowid AND m.AliquotCount = 0 AND m.rowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()); - new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); - - }); - } - else - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); - boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit - totalUnitsStr = volumeUnits.get(0).unit; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append("a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append("parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append("aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - service.getValidatedUnit("box", Unit.unit, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) + { + if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) + return false; + + boolean hasToValidateUnit = true; + Unit newUnit = Unit.fromName(newUnitStr); + if (!StringUtils.isEmpty(st.getMetricUnit())) + { + Unit oldUnit = Unit.fromName(st.getMetricUnit()); + if (oldUnit != null && newUnit != null) + hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); + } + + if (hasToValidateUnit) + { + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); + filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); + if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) + { + List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); + filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); + } + else + filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); + + TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); + return ts.exists(); + } + + return false; + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + + if (hasIncompatibleUnits(st, options.getMetricUnit())) + throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); + + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + Unit sampleTypeDisplayUnit = Unit.valueOf(sampleTypeUnit); + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = sampleTypeDisplayUnit.getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + + boolean isCountUnitType = sampleTypeBaseUnit.getKindOfQuantity() == KindOfQuantity.Count; + String aliquotUnitSql = isCountUnitType ? "CASE WHEN MIN(im.units) = MAX(im.units) THEN MIN(im.units) ELSE ? END" : "?"; + + SQLFragment statsSql = new SQLFragment("SELECT im.rootmaterialrowid, SUM(im.storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN im.samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN im.storedamount ELSE 0 END) AS avail_volume, \n") + .append(aliquotUnitSql) + .append(" AS common_unit \n").add(baseUnit) + .append("FROM exp.material im\n") + .append("WHERE im.rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND im.rowid != im.rootmaterialrowid\n") + .append(" GROUP BY im.rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = null; + + if (tableInfo.getSchema().getSqlDialect().isSqlServer()) + { + /* + * SqlServer needs to specify the alias in the FROM clause, and use that alias as the target of the update. + */ + quickRollUpSql = new SQLFragment("UPDATE exp.material SET \n") + .append("aliquotvolume = ROUND(CAST(COALESCE(stats.total_volume, 0) AS NUMERIC(18,6)) , ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(CAST(COALESCE(stats.avail_volume, 0) AS NUMERIC(18,6)), ?)\n").add(precisionScale) + .append("FROM exp.material m INNER JOIN (") + .append(statsSql) + .append(") AS stats\n") + .append("ON m.rowid = stats.rootmaterialrowid" + ); + } + else + { + /* + * Alias usage: PostgreSQL allows you to use an alias in the UPDATE clause itself + * Type casting: PostgreSQL uses ::NUMERIC for type casting. + * JOIN condition: The WHERE clause is used for joining the tables instead of an INNER JOIN with ON. + */ + quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + } + + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE rowid = rootmaterialrowid AND AliquotCount = 0 AND rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} From 40e2b234611be43538e7c4cf568e957bd40ee0c8 Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 1 Dec 2025 19:12:30 -0800 Subject: [PATCH 08/18] Validate amount range on bulk/detail form. Show delta warning for empty initial amount --- .../src/org/labkey/experiment/api/SampleTypeServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index e3434a9ac86..b1d86e0e454 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1592,9 +1592,9 @@ public int recomputeSamplesRollup( * SqlServer needs to specify the alias in the FROM clause, and use that alias as the target of the update. */ quickRollUpSql = new SQLFragment("UPDATE exp.material SET \n") - .append("aliquotvolume = ROUND(CAST(COALESCE(stats.total_volume, 0) AS NUMERIC(18,6)) , ?),\n").add(precisionScale) + .append("aliquotvolume = ROUND(CAST(COALESCE(stats.total_volume, 0) AS NUMERIC(38,12)) , ?),\n").add(precisionScale) .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(CAST(COALESCE(stats.avail_volume, 0) AS NUMERIC(18,6)), ?)\n").add(precisionScale) + .append("availablealiquotvolume = ROUND(CAST(COALESCE(stats.avail_volume, 0) AS NUMERIC(38,12)), ?)\n").add(precisionScale) .append("FROM exp.material m INNER JOIN (") .append(statsSql) .append(") AS stats\n") From 1e21a46771cb34808a263fafbd6449e6714d8713 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 2 Dec 2025 18:36:40 -0800 Subject: [PATCH 09/18] bug fixes and test updates --- experiment/src/org/labkey/experiment/ExperimentModule.java | 3 +++ .../src/org/labkey/experiment/api/SampleTypeServiceImpl.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 44879fe3998..b3610d22354 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -740,6 +740,9 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng', 'pg')").getObject(Long.class)); + results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?)", "unit").getObject(Long.class)); results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index b1d86e0e454..327eb1a7aa5 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1570,6 +1570,11 @@ public int recomputeSamplesRollup( return; int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + if (precisionScale > 9 && sampleTypeDisplayUnit.getValue() > 1e-9) + { + // reserve higher precisionScale for when display units are very small, like ng or pg + precisionScale = 9; + } boolean isCountUnitType = sampleTypeBaseUnit.getKindOfQuantity() == KindOfQuantity.Count; String aliquotUnitSql = isCountUnitType ? "CASE WHEN MIN(im.units) = MAX(im.units) THEN MIN(im.units) ELSE ? END" : "?"; From 345e495ee214011b89463fbf4679a57aada10552 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 2 Dec 2025 22:24:10 -0800 Subject: [PATCH 10/18] fix tests --- .../src/org/labkey/experiment/api/SampleTypeServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 327eb1a7aa5..582766832e7 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1085,6 +1085,8 @@ private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); } + else if (newUnit != null) + filter.addCondition(FieldKey.fromParts("Units"), newUnit.getBase().name(), CompareType.NEQ); else filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); From 59500c181aa8bf58891a79e8a041b564c9f77f1d Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 3 Dec 2025 09:40:17 -0800 Subject: [PATCH 11/18] CRLF --- .../experiment/api/SampleTypeServiceImpl.java | 4946 ++++++++--------- 1 file changed, 2473 insertions(+), 2473 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 582766832e7..db699694731 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2473 +1,2473 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - if (defaultUnits != null) - commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) - { - if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) - return false; - - boolean hasToValidateUnit = true; - Unit newUnit = Unit.fromName(newUnitStr); - if (!StringUtils.isEmpty(st.getMetricUnit())) - { - Unit oldUnit = Unit.fromName(st.getMetricUnit()); - if (oldUnit != null && newUnit != null) - hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); - } - - if (hasToValidateUnit) - { - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); - filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); - if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) - { - List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); - filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); - } - else if (newUnit != null) - filter.addCondition(FieldKey.fromParts("Units"), newUnit.getBase().name(), CompareType.NEQ); - else - filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); - - TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); - return ts.exists(); - } - - return false; - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - - if (hasIncompatibleUnits(st, options.getMetricUnit())) - throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); - - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - if (!StringUtils.isEmpty(sampleTypeUnit)) - { - Unit sampleTypeDisplayUnit = Unit.valueOf(sampleTypeUnit); - // if sample type has unit, use it for simple rollup without need for conversion - Unit sampleTypeBaseUnit = sampleTypeDisplayUnit.getBase(); - String baseUnit = sampleTypeBaseUnit.name(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> - { - if (sublist.isEmpty()) - return; - - int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); - if (precisionScale > 9 && sampleTypeDisplayUnit.getValue() > 1e-9) - { - // reserve higher precisionScale for when display units are very small, like ng or pg - precisionScale = 9; - } - - boolean isCountUnitType = sampleTypeBaseUnit.getKindOfQuantity() == KindOfQuantity.Count; - String aliquotUnitSql = isCountUnitType ? "CASE WHEN MIN(im.units) = MAX(im.units) THEN MIN(im.units) ELSE ? END" : "?"; - - SQLFragment statsSql = new SQLFragment("SELECT im.rootmaterialrowid, SUM(im.storedamount) AS total_volume, \n") - .append("SUM(CASE WHEN im.samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN im.storedamount ELSE 0 END) AS avail_volume, \n") - .append(aliquotUnitSql) - .append(" AS common_unit \n").add(baseUnit) - .append("FROM exp.material im\n") - .append("WHERE im.rootmaterialrowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()) - .append(" AND im.rowid != im.rootmaterialrowid\n") - .append(" GROUP BY im.rootmaterialrowid\n"); - - SQLFragment quickRollUpSql = null; - - if (tableInfo.getSchema().getSqlDialect().isSqlServer()) - { - /* - * SqlServer needs to specify the alias in the FROM clause, and use that alias as the target of the update. - */ - quickRollUpSql = new SQLFragment("UPDATE exp.material SET \n") - .append("aliquotvolume = ROUND(CAST(COALESCE(stats.total_volume, 0) AS NUMERIC(38,12)) , ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(CAST(COALESCE(stats.avail_volume, 0) AS NUMERIC(38,12)), ?)\n").add(precisionScale) - .append("FROM exp.material m INNER JOIN (") - .append(statsSql) - .append(") AS stats\n") - .append("ON m.rowid = stats.rootmaterialrowid" - ); - } - else - { - /* - * Alias usage: PostgreSQL allows you to use an alias in the UPDATE clause itself - * Type casting: PostgreSQL uses ::NUMERIC for type casting. - * JOIN condition: The WHERE clause is used for joining the tables instead of an INNER JOIN with ON. - */ - quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") - .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) - .append("aliquotunit = stats.common_unit,\n") - .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) - .append("FROM (") - .append(statsSql) - .append(") AS stats\n") - .append("WHERE m.rowid = stats.rootmaterialrowid" - ); - } - - new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); - - // Now clear out rollups for samples that have zero aliquots - SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material SET \n") - .append("aliquotvolume = 0, availablealiquotvolume = 0, ") - .append("aliquotunit = ?\n").add(baseUnit) - .append("WHERE rowid = rootmaterialrowid AND AliquotCount = 0 AND rowid ") - .appendInClause(sublist, tableInfo.getSqlDialect()); - new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); - - }); - } - else - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); - boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit - totalUnitsStr = volumeUnits.get(0).unit; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append("a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - SQLFragment sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append("parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append("aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - service.getValidatedUnit("box", Unit.unit, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + if (defaultUnits != null) + commonUnits = commonUnits.stream().filter(u -> u.getKindOfQuantity() == defaultUnits.getKindOfQuantity()).collect(Collectors.toList()); + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + private boolean hasIncompatibleUnits(ExpSampleTypeImpl st, String newUnitStr) + { + if (StringUtils.isEmpty(newUnitStr) || newUnitStr.equalsIgnoreCase(st.getMetricUnit())) + return false; + + boolean hasToValidateUnit = true; + Unit newUnit = Unit.fromName(newUnitStr); + if (!StringUtils.isEmpty(st.getMetricUnit())) + { + Unit oldUnit = Unit.fromName(st.getMetricUnit()); + if (oldUnit != null && newUnit != null) + hasToValidateUnit = !oldUnit.getBase().equals(newUnit.getBase()); + } + + if (hasToValidateUnit) + { + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("CpasType"), st.getLSID()); + filter.addCondition(FieldKey.fromParts("StoredAmount"), null, CompareType.NONBLANK); + if (newUnit != null && newUnit.getBase() == Unit.unit.getBase()) + { + List compatibleUnits = KindOfQuantity.Count.getCommonUnits().stream().map(Unit::name).collect(Collectors.toList()); + filter.addCondition(FieldKey.fromParts("Units"), compatibleUnits, CompareType.NOT_IN); + } + else if (newUnit != null) + filter.addCondition(FieldKey.fromParts("Units"), newUnit.getBase().name(), CompareType.NEQ); + else + filter.addCondition(FieldKey.fromParts("Units"), newUnitStr, CompareType.NEQ); + + TableSelector ts = new TableSelector(getTinfoMaterial(), filter, null); + return ts.exists(); + } + + return false; + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + + if (hasIncompatibleUnits(st, options.getMetricUnit())) + throw new ApiUsageException("Unable to update 'Display Units' to '" + options.getMetricUnit() + "'. There are existing samples with incompatible units."); + + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + if (!StringUtils.isEmpty(sampleTypeUnit)) + { + Unit sampleTypeDisplayUnit = Unit.valueOf(sampleTypeUnit); + // if sample type has unit, use it for simple rollup without need for conversion + Unit sampleTypeBaseUnit = sampleTypeDisplayUnit.getBase(); + String baseUnit = sampleTypeBaseUnit.name(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + { + if (sublist.isEmpty()) + return; + + int precisionScale = sampleTypeBaseUnit.getPrecisionScale(); + if (precisionScale > 9 && sampleTypeDisplayUnit.getValue() > 1e-9) + { + // reserve higher precisionScale for when display units are very small, like ng or pg + precisionScale = 9; + } + + boolean isCountUnitType = sampleTypeBaseUnit.getKindOfQuantity() == KindOfQuantity.Count; + String aliquotUnitSql = isCountUnitType ? "CASE WHEN MIN(im.units) = MAX(im.units) THEN MIN(im.units) ELSE ? END" : "?"; + + SQLFragment statsSql = new SQLFragment("SELECT im.rootmaterialrowid, SUM(im.storedamount) AS total_volume, \n") + .append("SUM(CASE WHEN im.samplestate ").appendInClause(availableSampleStates, tableInfo.getSqlDialect()).append(" THEN im.storedamount ELSE 0 END) AS avail_volume, \n") + .append(aliquotUnitSql) + .append(" AS common_unit \n").add(baseUnit) + .append("FROM exp.material im\n") + .append("WHERE im.rootmaterialrowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()) + .append(" AND im.rowid != im.rootmaterialrowid\n") + .append(" GROUP BY im.rootmaterialrowid\n"); + + SQLFragment quickRollUpSql = null; + + if (tableInfo.getSchema().getSqlDialect().isSqlServer()) + { + /* + * SqlServer needs to specify the alias in the FROM clause, and use that alias as the target of the update. + */ + quickRollUpSql = new SQLFragment("UPDATE exp.material SET \n") + .append("aliquotvolume = ROUND(CAST(COALESCE(stats.total_volume, 0) AS NUMERIC(38,12)) , ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(CAST(COALESCE(stats.avail_volume, 0) AS NUMERIC(38,12)), ?)\n").add(precisionScale) + .append("FROM exp.material m INNER JOIN (") + .append(statsSql) + .append(") AS stats\n") + .append("ON m.rowid = stats.rootmaterialrowid" + ); + } + else + { + /* + * Alias usage: PostgreSQL allows you to use an alias in the UPDATE clause itself + * Type casting: PostgreSQL uses ::NUMERIC for type casting. + * JOIN condition: The WHERE clause is used for joining the tables instead of an INNER JOIN with ON. + */ + quickRollUpSql = new SQLFragment("UPDATE exp.material AS m SET \n") + .append("aliquotvolume = ROUND(COALESCE(stats.total_volume, 0)::NUMERIC, ?),\n").add(precisionScale) + .append("aliquotunit = stats.common_unit,\n") + .append("availablealiquotvolume = ROUND(COALESCE(stats.avail_volume, 0)::NUMERIC, ?)\n").add(precisionScale) + .append("FROM (") + .append(statsSql) + .append(") AS stats\n") + .append("WHERE m.rowid = stats.rootmaterialrowid" + ); + } + + new SqlExecutor(tableInfo.getSchema()).execute(quickRollUpSql); + + // Now clear out rollups for samples that have zero aliquots + SQLFragment quickClearRollupSql = new SQLFragment("UPDATE exp.material SET \n") + .append("aliquotvolume = 0, availablealiquotvolume = 0, ") + .append("aliquotunit = ?\n").add(baseUnit) + .append("WHERE rowid = rootmaterialrowid AND AliquotCount = 0 AND rowid ") + .appendInClause(sublist, tableInfo.getSqlDialect()); + new SqlExecutor(tableInfo.getSchema()).execute(quickClearRollupSql); + + }); + } + else + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Set uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; + + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (hasSameAliquotUnit && !StringUtils.isEmpty(volumeUnits.get(0).unit)) // if all aliquots have the same unit, prefer it over parent's unit + totalUnitsStr = volumeUnits.get(0).unit; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append("a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + SQLFragment sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append("parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append("aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + service.getValidatedUnit("box", Unit.unit, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} From 49941559c2ba71065ffd4ae7918da2791e61a95a Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 3 Dec 2025 12:19:50 -0800 Subject: [PATCH 12/18] fix metrics --- experiment/src/org/labkey/experiment/ExperimentModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 94bf10d8f3d..b04ead24a1b 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -742,7 +742,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng', 'pg')").getObject(Long.class)); results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); - results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?)", "unit").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); From d93154baadb995cdfd73490b2843c44435a154e3 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 3 Dec 2025 14:36:56 -0800 Subject: [PATCH 13/18] remove support for pg unit --- api/src/org/labkey/api/ontology/KindOfQuantity.java | 2 +- .../src/client/test/integration/SampleTypeCrud.ispec.ts | 7 +------ experiment/src/org/labkey/experiment/ExperimentModule.java | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java index 4c7f3e2568e..dc249c3e010 100644 --- a/api/src/org/labkey/api/ontology/KindOfQuantity.java +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -29,7 +29,7 @@ public List getCommonUnits() @Override public List getCommonUnits() { - return List.of(Unit.kg, Unit.g, Unit.mg, Unit.ug, Unit.ng, Unit.pg); + return List.of(Unit.kg, Unit.g, Unit.mg, Unit.ug, Unit.ng); } }, diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 84a02d9eee1..2f715a2c34f 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -979,7 +979,7 @@ describe('Amount/Unit CRUD', () => { errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tcells", dataType, "INSERT", topFolderOptions, editorUserOptions); expect(errorMsg.text).toContain('Units value (cells) is not compatible with the ' + dataType + ' display units (g).'); errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t1.1\tbogus", dataType, "INSERT", topFolderOptions, editorUserOptions); - expect(errorMsg.text).toContain('Unsupported Units value (bogus). Supported values are: kg, g, mg, ug, ng, pg.'); + expect(errorMsg.text).toContain('Unsupported Units value (bogus). Supported values are: kg, g, mg, ug, ng.'); errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\nData1\t-1.1\tkg", dataType, "INSERT", topFolderOptions, editorUserOptions); expect(errorMsg.text).toContain(NEGATIVE_ERROR); errorMsg = await ExperimentCRUDUtils.importCrossTypeData(server, "Name\tStoredAmount\tUnits\tSampleType\nData1\t-1.1\tkg\t" + dataType ,'IMPORT', topFolderOptions, adminOptions, true); @@ -1115,7 +1115,6 @@ describe('Amount/Unit CRUD', () => { } let sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, [ - {name: 'S-pg', amount: 4.56, units: 'pg'}, {name: 'S-ng', amount: 4.56, units: 'ng'}, {name: 'S-ug', amount: 4.56, units: 'ug'}, {name: 'S-mg', amount: 4.56, units: 'mg'}, @@ -1125,7 +1124,6 @@ describe('Amount/Unit CRUD', () => { // check for storedamount in g let expectedRawAmounts : {} = { - 'S-pg': 4.56e-12, 'S-ng': 4.56e-9, 'S-ug': 4.56e-6, 'S-mg': 0.00456, @@ -1133,7 +1131,6 @@ describe('Amount/Unit CRUD', () => { 'S-kg': 4560, }; let expectedStoredAmounts : {} = { - 'S-pg': 4.56e-6, 'S-ng': 4.56e-3, 'S-ug': 4.56, 'S-mg': 4560, @@ -1160,7 +1157,6 @@ describe('Amount/Unit CRUD', () => { } expectedRawAmounts = { - 'S-pg': 6.54e-12, 'S-ng': 6.54e-9, 'S-ug': 6.54e-6, 'S-mg': 0.00654, @@ -1168,7 +1164,6 @@ describe('Amount/Unit CRUD', () => { 'S-kg': 6540, }; expectedStoredAmounts = { - 'S-pg': 6.54e-6, 'S-ng': 6.54e-3, 'S-ug': 6.54, 'S-mg': 6540, diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index b04ead24a1b..404841d1f3e 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -740,7 +740,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng', 'pg')").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); From 59893fdb0d614cff732a151cf99ce17b161d8437 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 3 Dec 2025 14:41:09 -0800 Subject: [PATCH 14/18] clean --- .../src/org/labkey/experiment/api/SampleTypeServiceImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index db699694731..86509da4546 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1566,7 +1566,7 @@ public int recomputeSamplesRollup( TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> + ListUtils.partition(new ArrayList<>(withAmountsParents), 1000).forEach(sublist -> { if (sublist.isEmpty()) return; @@ -1718,10 +1718,9 @@ private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List uniqueAliquotUnits = volumeUnits.stream() .map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); + Set uniqueAliquotUnits = volumeUnits.stream().map(AliquotAmountUnitResult::unit).collect(Collectors.toSet()); boolean hasSameAliquotUnit = uniqueAliquotUnits.size() <= 1; - Unit totalUnit = null; String totalUnitsStr; if (!StringUtils.isEmpty(sampleTypeUnitsStr)) From d87cd5463f7ca9a12d21ec75d5f94666f2e1faa7 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 4 Dec 2025 21:19:50 -0800 Subject: [PATCH 15/18] bug fixes --- .../api/exp/query/ExpMaterialTable.java | 6 ++-- .../labkey/api/ontology/KindOfQuantity.java | 2 +- api/src/org/labkey/api/ontology/Quantity.java | 2 +- api/src/org/labkey/api/ontology/Unit.java | 25 ++++++++------ .../test/integration/SampleTypeCrud.ispec.ts | 30 ++++++++-------- .../experiment/api/ExpMaterialTableImpl.java | 34 +++++++++++-------- 6 files changed, 53 insertions(+), 46 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java index 82b19f98578..0956af9d020 100644 --- a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java +++ b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java @@ -52,9 +52,9 @@ enum Column Properties, Property, QueryableInputs, - RawAliquotUnit, - RawAliquotVolume, - RawAvailableAliquotVolume, + RawAliquotUnit(false, "Raw Aliquot Unit"), + RawAliquotVolume(false, "Raw Aliquot Total Amount"), + RawAvailableAliquotVolume(false, "Raw Available Aliquot Amount"), RawAmount(true), RawUnits, RootMaterialRowId, diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java index dc249c3e010..e61bb42169a 100644 --- a/api/src/org/labkey/api/ontology/KindOfQuantity.java +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -39,7 +39,7 @@ public List getCommonUnits() @Override public List getCommonUnits() { - return List.of(Unit.unit, Unit.blocks, Unit.bottle, Unit.box, Unit.cells, Unit.kit, Unit.pack, Unit.pcs, Unit.slides, Unit.tests); + return List.of(Unit.blocks, Unit.bottles, Unit.boxes, Unit.cells, Unit.kits, Unit.packs, Unit.pieces, Unit.slides, Unit.tests, Unit.unit); } }; diff --git a/api/src/org/labkey/api/ontology/Quantity.java b/api/src/org/labkey/api/ontology/Quantity.java index aa4c54ea363..54e5f33ae48 100644 --- a/api/src/org/labkey/api/ontology/Quantity.java +++ b/api/src/org/labkey/api/ontology/Quantity.java @@ -465,7 +465,7 @@ public void testParse() assertEquals(Quantity.of(0, Unit.count), parse("0 units")); assertEquals(Quantity.of(0, Unit.count), parse("0count")); - assertEquals(Quantity.of(1, Unit.count), parse("1", Unit.box)); + assertEquals(Quantity.of(1, Unit.count), parse("1", Unit.boxes)); assertEquals(Quantity.of(1, Unit.unit), parse("1", Unit.blocks)); assertEquals(Quantity.of(1, Unit.cells), parse("1", Unit.tests)); diff --git a/api/src/org/labkey/api/ontology/Unit.java b/api/src/org/labkey/api/ontology/Unit.java index ade6e995bd6..6a42d2739d5 100644 --- a/api/src/org/labkey/api/ontology/Unit.java +++ b/api/src/org/labkey/api/ontology/Unit.java @@ -19,10 +19,10 @@ public enum Unit count(KindOfQuantity.Count, unit, 1.0, 2, "count", Quantity.class, "count", "count"), - pcs(KindOfQuantity.Count, unit, 1.0, 2, "pcs", + pieces(KindOfQuantity.Count, unit, 1.0, 2, "pieces", Quantity.class, - "pcs", "pcs"), - pack(KindOfQuantity.Count, unit, 1.0, 2, "pack", + "piece", "pieces"), + packs(KindOfQuantity.Count, unit, 1.0, 2, "packs", Quantity.class, "pack", "packs"), blocks(KindOfQuantity.Count, unit, 1.0, 2, "blocks", @@ -34,16 +34,16 @@ public enum Unit cells(KindOfQuantity.Count, unit, 1.0, 2, "cells", Quantity.class, "cell", "cells"), - box(KindOfQuantity.Count, unit, 1.0, 2, "box", + boxes(KindOfQuantity.Count, unit, 1.0, 2, "box", Quantity.class, "box", "boxes"), - kit(KindOfQuantity.Count, unit, 1.0, 2, "kit", + kits(KindOfQuantity.Count, unit, 1.0, 2, "kits", Quantity.class, "kit", "kits"), tests(KindOfQuantity.Count, unit, 1.0, 2, "tests", Quantity.class, "test", "tests"), - bottle(KindOfQuantity.Count, unit, 1.0, 2, "bottle", + bottles(KindOfQuantity.Count, unit, 1.0, 2, "bottles", Quantity.class, "bottle", "bottles"), @@ -233,7 +233,7 @@ public void testIsBase() assertFalse(Unit.kg.isBase()); assertTrue(Unit.unit.isBase()); assertFalse(Unit.count.isBase()); - assertFalse(Unit.bottle.isBase()); + assertFalse(Unit.bottles.isBase()); } @Test @@ -245,13 +245,16 @@ public void testIsCompatible() assertTrue(Unit.g.isCompatible(Unit.mg)); assertFalse(Unit.g.isCompatible(Unit.mL)); assertTrue(Unit.unit.isCompatible(Unit.count)); - assertTrue(Unit.unit.isCompatible(Unit.pcs)); - assertTrue(Unit.unit.isCompatible(Unit.pack)); - assertTrue(Unit.unit.isCompatible(Unit.bottle)); + assertTrue(Unit.unit.isCompatible(Unit.pieces)); + assertTrue(Unit.unit.isCompatible(Unit.packs)); + assertTrue(Unit.unit.isCompatible(Unit.bottles)); assertTrue(Unit.unit.isCompatible(Unit.blocks)); - assertTrue(Unit.unit.isCompatible(Unit.box)); + assertTrue(Unit.unit.isCompatible(Unit.boxes)); assertTrue(Unit.unit.isCompatible(Unit.slides)); + assertTrue(Unit.cells.isCompatible(Unit.slides)); + assertTrue(Unit.cells.isCompatible(Unit.unit)); assertFalse(Unit.unit.isCompatible(Unit.mL)); + assertFalse(Unit.bottles.isCompatible(Unit.mL)); assertFalse(Unit.mL.isCompatible(null)); } diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 2f715a2c34f..c1968cc00e4 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -1122,7 +1122,7 @@ describe('Amount/Unit CRUD', () => { {name: 'S-kg', amount: 4.56, units: 'kg'}, ], 'samples', sampleTypeMass, topFolderOptions, editorUserOptions); - // check for storedamount in g + // check for raw amount in g and display amount in ug let expectedRawAmounts : {} = { 'S-ng': 4.56e-9, 'S-ug': 4.56e-6, @@ -1208,8 +1208,8 @@ describe('Amount/Unit CRUD', () => { const countRows = [ {name: 'S-unit', amount: 4.56, units: 'unit'}, - {name: 'S-pcs', amount: 4.56, units: 'pcs'}, - {name: 'S-kit', amount: 4.56, units: 'kit'}, + {name: 'S-pcs', amount: 4.56, units: 'pieces'}, + {name: 'S-kit', amount: 4.56, units: 'kits'}, {name: 'S-cells', amount: 4.56, units: 'cells'} ] sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, countRows, 'samples', sampleTypeCount, topFolderOptions, editorUserOptions); @@ -1249,19 +1249,19 @@ describe('Amount/Unit CRUD', () => { async function verifyCountTypeAliquotRollup(sampleTypeName: string, hasSampleTypeDisplayUnit: boolean) { const dataRows = [ {name: 'S-no-amount'}, - {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs1', amount: 2, units: 'pcs'}, - {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs2', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs1', amount: 2, units: 'pieces'}, + {AliquotedFrom: 'S-no-amount', name: 'S-no-pcs2', amount: 2, units: 'pieces'}, {name: 'S-unit', amount: 1, units: 'unit'}, {AliquotedFrom: 'S-unit', name: 'S-unit-unit1', amount: 2, units: 'unit'}, {AliquotedFrom: 'S-unit', name: 'S-unit-unit2', amount: 2, units: 'unit'}, - {name: 'S-pcs', amount: 1, units: 'pcs'}, - {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs1', amount: 2, units: 'pcs'}, - {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs2', amount: 2, units: 'pcs'}, - {name: 'S-kit', amount: 1, units: 'kit'}, - {AliquotedFrom: 'S-kit', name: 'S-kit-pcs1', amount: 2, units: 'pcs'}, - {AliquotedFrom: 'S-kit', name: 'S-kit-pcs2', amount: 2, units: 'pcs'}, + {name: 'S-pcs', amount: 1, units: 'pieces'}, + {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs1', amount: 2, units: 'pieces'}, + {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs2', amount: 2, units: 'pieces'}, + {name: 'S-kit', amount: 1, units: 'kits'}, + {AliquotedFrom: 'S-kit', name: 'S-kit-pcs1', amount: 2, units: 'pieces'}, + {AliquotedFrom: 'S-kit', name: 'S-kit-pcs2', amount: 2, units: 'pieces'}, {name: 'S-cells', amount: 1, units: 'cells'}, - {AliquotedFrom: 'S-cells', name: 'S-cells-pcs1', amount: 2, units: 'pcs'}, + {AliquotedFrom: 'S-cells', name: 'S-cells-pcs1', amount: 2, units: 'pieces'}, {AliquotedFrom: 'S-cells', name: 'S-cells-cells2', amount: 2, units: 'cells'}, ] @@ -1272,10 +1272,10 @@ describe('Amount/Unit CRUD', () => { } let expectedAliquotUnit = { - 'S-no-amount': 'pcs', + 'S-no-amount': 'pieces', 'S-unit': 'unit', - 'S-pcs': 'pcs', - 'S-kit': 'pcs', + 'S-pcs': 'pieces', + 'S-kit': 'pieces', 'S-cells': hasSampleTypeDisplayUnit ? 'unit' : 'cells', }; diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index ce36d961059..89f4954b056 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -149,9 +149,13 @@ import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotUnit; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAliquotUnit; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAvailableAliquotVolume; import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; @@ -569,7 +573,7 @@ public StringExpression getURL(ColumnInfo parent) case RawAliquotVolume -> { var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); - ret.setLabel("Raw " + ALIQUOT_VOLUME_LABEL); + ret.setLabel(RawAliquotVolume.label()); ret.setShownInDetailsView(false); return ret; } @@ -578,7 +582,7 @@ public StringExpression getURL(ColumnInfo parent) Unit typeUnit = getSampleTypeUnit(); if (typeUnit != null) { - SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AliquotVolume.name(), Column.AliquotUnit.name(), ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AliquotVolume.name(), AliquotUnit.name(), ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); columnInfo.setDescription("The total amount of this sample's aliquots, in the display unit for the sample type, currently on hand."); return columnInfo; @@ -593,7 +597,7 @@ public StringExpression getURL(ColumnInfo parent) case RawAvailableAliquotVolume -> { var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); - ret.setLabel("Raw " + AVAILABLE_ALIQUOT_VOLUME_LABEL); + ret.setLabel(RawAvailableAliquotVolume.label()); ret.setShownInDetailsView(false); return ret; } @@ -602,9 +606,9 @@ public StringExpression getURL(ColumnInfo parent) Unit typeUnit = getSampleTypeUnit(); if (typeUnit != null) { - SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AvailableAliquotVolume.name(), Column.AliquotUnit.name(), AVAILABLE_ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.AvailableAliquotVolume.name(), AliquotUnit.name(), AVAILABLE_ALIQUOT_VOLUME_LABEL, Collections.emptySet(), typeUnit); columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); - columnInfo.setDescription("The total amount of this sample's aliquots that's available, in the display unit for the sample type, currently on hand."); + columnInfo.setDescription("The total amount of this sample's available aliquots currently on hand, in the display unit for the sample type."); return columnInfo; } else @@ -624,7 +628,7 @@ public StringExpression getURL(ColumnInfo parent) { var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); ret.setShownInDetailsView(false); - ret.setLabel("Raw Aliquot Unit"); + ret.setLabel(RawAliquotUnit.label()); return ret; } case AliquotUnit -> @@ -641,7 +645,7 @@ public StringExpression getURL(ColumnInfo parent) Unit typeUnit = getSampleTypeUnit(); if (typeUnit != null) { - SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.AliquotUnit.name(), typeUnit); + SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, AliquotUnit.name(), typeUnit); columnInfo.setFk(fk); columnInfo.setDescription("The sample type display units associated with the AliquotAmount for this sample."); return columnInfo; @@ -888,7 +892,7 @@ protected ContainerFilter getLookupContainerFilter() addColumn(Column.AliquotCount); addColumn(Column.AliquotVolume); - addColumn(Column.AliquotUnit); + addColumn(AliquotUnit); addColumn(Column.AvailableAliquotCount); addColumn(Column.AvailableAliquotVolume); @@ -944,7 +948,7 @@ public void addQueryFieldKeys(Set keys) rawUnitsColumn.setShownInInsertView(false); rawUnitsColumn.setShownInUpdateView(false); - var rawAliquotVolumeColumn = addColumn(Column.RawAliquotVolume); + var rawAliquotVolumeColumn = addColumn(RawAliquotVolume); rawAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() { @Override @@ -956,7 +960,7 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) public void addQueryFieldKeys(Set keys) { super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(AliquotVolume)); + keys.add(AliquotVolume.fieldKey()); } }; @@ -967,7 +971,7 @@ public void addQueryFieldKeys(Set keys) rawAliquotVolumeColumn.setShownInInsertView(false); rawAliquotVolumeColumn.setShownInUpdateView(false); - var rawAvailableAliquotVolumeColumn = addColumn(Column.RawAvailableAliquotVolume); + var rawAvailableAliquotVolumeColumn = addColumn(RawAvailableAliquotVolume); rawAvailableAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() { @Override @@ -979,7 +983,7 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) public void addQueryFieldKeys(Set keys) { super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(AvailableAliquotVolume)); + keys.add(AvailableAliquotVolume.fieldKey()); } }; @@ -990,7 +994,7 @@ public void addQueryFieldKeys(Set keys) rawAvailableAliquotVolumeColumn.setShownInInsertView(false); rawAvailableAliquotVolumeColumn.setShownInUpdateView(false); - var rawAliquotUnitColumn = addColumn(Column.RawAliquotUnit); + var rawAliquotUnitColumn = addColumn(RawAliquotUnit); rawAliquotUnitColumn.setDisplayColumnFactory(new DisplayColumnFactory() { @Override @@ -1002,7 +1006,7 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) public void addQueryFieldKeys(Set keys) { super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.AliquotUnit)); + keys.add(AliquotUnit.fieldKey()); } }; @@ -1265,7 +1269,7 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); if (selectedColumns.contains(new FieldKey(null, AliquotVolume.name())) || selectedColumns.contains(new FieldKey(null, AvailableAliquotVolume.name()))) - selectedColumns.add(new FieldKey(null, Column.AliquotUnit.name())); + selectedColumns.add(new FieldKey(null, AliquotUnit.name())); selectedColumns.addAll(wrappedFieldKeys); if (null != getFilter()) selectedColumns.addAll(getFilter().getAllFieldKeys()); From 23d06d357e40ee5b25184088ef1bf8d5218516ba Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 5 Dec 2025 10:09:36 -0800 Subject: [PATCH 16/18] fix tests --- .../test/integration/SampleTypeCrud.ispec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index c1968cc00e4..5f637c9f521 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -1254,12 +1254,12 @@ describe('Amount/Unit CRUD', () => { {name: 'S-unit', amount: 1, units: 'unit'}, {AliquotedFrom: 'S-unit', name: 'S-unit-unit1', amount: 2, units: 'unit'}, {AliquotedFrom: 'S-unit', name: 'S-unit-unit2', amount: 2, units: 'unit'}, - {name: 'S-pcs', amount: 1, units: 'pieces'}, - {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs1', amount: 2, units: 'pieces'}, - {AliquotedFrom: 'S-pcs', name: 'S-pcs-pcs2', amount: 2, units: 'pieces'}, - {name: 'S-kit', amount: 1, units: 'kits'}, - {AliquotedFrom: 'S-kit', name: 'S-kit-pcs1', amount: 2, units: 'pieces'}, - {AliquotedFrom: 'S-kit', name: 'S-kit-pcs2', amount: 2, units: 'pieces'}, + {name: 'S-pieces', amount: 1, units: 'pieces'}, + {AliquotedFrom: 'S-pieces', name: 'S-pcs-pcs1', amount: 2, units: 'pieces'}, + {AliquotedFrom: 'S-pieces', name: 'S-pcs-pcs2', amount: 2, units: 'pieces'}, + {name: 'S-kits', amount: 1, units: 'kits'}, + {AliquotedFrom: 'S-kits', name: 'S-kit-pcs1', amount: 2, units: 'pieces'}, + {AliquotedFrom: 'S-kits', name: 'S-kit-pcs2', amount: 2, units: 'pieces'}, {name: 'S-cells', amount: 1, units: 'cells'}, {AliquotedFrom: 'S-cells', name: 'S-cells-pcs1', amount: 2, units: 'pieces'}, {AliquotedFrom: 'S-cells', name: 'S-cells-cells2', amount: 2, units: 'cells'}, @@ -1274,8 +1274,8 @@ describe('Amount/Unit CRUD', () => { let expectedAliquotUnit = { 'S-no-amount': 'pieces', 'S-unit': 'unit', - 'S-pcs': 'pieces', - 'S-kit': 'pieces', + 'S-pieces': 'pieces', + 'S-kits': 'pieces', 'S-cells': hasSampleTypeDisplayUnit ? 'unit' : 'cells', }; From 30c5a6dc4bf3688778e324b24d16b14c3ef33fe8 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 5 Dec 2025 14:08:19 -0800 Subject: [PATCH 17/18] fix updating other unit from sample detail panel --- .../src/client/test/integration/SampleTypeCrud.ispec.ts | 4 ++-- .../src/org/labkey/experiment/api/SampleTypeServiceImpl.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 5f637c9f521..46c70b1059f 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -1208,8 +1208,8 @@ describe('Amount/Unit CRUD', () => { const countRows = [ {name: 'S-unit', amount: 4.56, units: 'unit'}, - {name: 'S-pcs', amount: 4.56, units: 'pieces'}, - {name: 'S-kit', amount: 4.56, units: 'kits'}, + {name: 'S-pieces', amount: 4.56, units: 'pieces'}, + {name: 'S-kits', amount: 4.56, units: 'kits'}, {name: 'S-cells', amount: 4.56, units: 'cells'} ] sampleRowsWithUnits = await ExperimentCRUDUtils.insertRows(server, countRows, 'samples', sampleTypeCount, topFolderOptions, editorUserOptions); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 86509da4546..17227bfb785 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1764,6 +1764,7 @@ else if (!StringUtils.isEmpty(sampleItemUnitsStr)) } catch (IllegalArgumentException ignore) { + // if aliquot units are incompatible, skip } double convertedAmount = 0; From d98e74ab0a1b35db5bc02279827edceb3bfdd1f0 Mon Sep 17 00:00:00 2001 From: XingY Date: Sat, 6 Dec 2025 22:06:08 -0800 Subject: [PATCH 18/18] fix more tests --- api/src/org/labkey/api/ontology/Unit.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/ontology/Unit.java b/api/src/org/labkey/api/ontology/Unit.java index 6a42d2739d5..b2872362cb7 100644 --- a/api/src/org/labkey/api/ontology/Unit.java +++ b/api/src/org/labkey/api/ontology/Unit.java @@ -34,7 +34,7 @@ public enum Unit cells(KindOfQuantity.Count, unit, 1.0, 2, "cells", Quantity.class, "cell", "cells"), - boxes(KindOfQuantity.Count, unit, 1.0, 2, "box", + boxes(KindOfQuantity.Count, unit, 1.0, 2, "boxes", Quantity.class, "box", "boxes"), kits(KindOfQuantity.Count, unit, 1.0, 2, "kits",