diff --git a/api/src/org/labkey/api/data/BaseColumnInfo.java b/api/src/org/labkey/api/data/BaseColumnInfo.java index cf5267571ca..30bf9f07046 100644 --- a/api/src/org/labkey/api/data/BaseColumnInfo.java +++ b/api/src/org/labkey/api/data/BaseColumnInfo.java @@ -46,6 +46,8 @@ import org.labkey.api.query.QueryParseException; import org.labkey.api.query.SchemaKey; import org.labkey.api.query.column.BuiltInColumnTypes; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringExpression; import org.labkey.api.util.StringExpressionFactory; @@ -1353,12 +1355,21 @@ public void setSortFieldKeysFromXml(String xml) public static String labelFromName(String name) { - if (name == null) - return null; - - if (name.isEmpty()) + if (StringUtils.isBlank(name)) return name; + // NOTE: This is just for testing (let the DataRegion/DataColumn do this) + if (OptionalFeatureService.get().isFeatureEnabled(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING)) + { + var index = name.indexOf("__"); + if (index > 0) + { + String unit = name.substring(index + 2); + if (null != org.labkey.api.ontology.Unit.fromName(unit)) + name = name.substring(0, index); + } + } + StringBuilder buf = new StringBuilder(name.length() + 10); char[] chars = new char[name.length()]; name.getChars(0, name.length(), chars, 0); diff --git a/api/src/org/labkey/api/data/ColumnRenderProperties.java b/api/src/org/labkey/api/data/ColumnRenderProperties.java index 20dde67a2d0..57b2a8ec140 100644 --- a/api/src/org/labkey/api/data/ColumnRenderProperties.java +++ b/api/src/org/labkey/api/data/ColumnRenderProperties.java @@ -15,19 +15,28 @@ */ package org.labkey.api.data; +import org.apache.commons.beanutils.ConvertUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.exp.PropertyType; import org.labkey.api.gwt.client.DefaultScaleType; import org.labkey.api.gwt.client.FacetingBehaviorType; +import org.labkey.api.ontology.KindOfQuantity; import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.util.StringExpression; +import org.labkey.api.util.logging.LogHelper; import java.io.File; import java.math.BigDecimal; +import java.text.DecimalFormatSymbols; import java.util.Date; import java.util.Set; +import java.util.function.Function; import static org.labkey.api.ontology.OntologyService.conceptCodeConceptURI; @@ -199,7 +208,7 @@ else if (Date.class.isAssignableFrom(javaClass)) */ boolean isScannable(); - /* Properties loaded by OntologyService */ + /* Properties loaded by OntologyService */ // any column can be annotated with PrincipalConceptCode default String getPrincipalConceptCode() @@ -245,6 +254,39 @@ default String getConceptLabelColumn() { return null; } + + default KindOfQuantity getKindOfQuantity() + { + var unit = getDisplayUnit(); + if (null == unit) + return null; + return unit.getKindOfQuantity(); + } + + default Unit getDisplayUnit() + { + if (OptionalFeatureService.get().isFeatureEnabled(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING)) + { + if (!getJdbcType().isNumeric()) + return null; + String name = getName(); + var index = name.lastIndexOf("__"); + if (index < 0) + return null; + var unitPart = name.substring(index + 2); + try + { + return Unit.valueOf(unitPart); + } + catch (IllegalArgumentException x) + { + // pass + } + } + return null; + } + + /* End properties loaded by OntologyService */ @@ -252,4 +294,105 @@ default String getDerivationDataScope() { return null; } + + + /** + * This Format can be used for low-level conversion of the type represented by this column. It does handle + * basic numeric/date conversion including formats, and default display unit handling. That's about it. + * It never produces HTML + * It does not handle compound/column formatting (missing values, oor, etc) + * It does not handle conditional formatting + * This method moves (most) of the work formerly done in DisplayColumn.formatValue() to a shared location (getFormat()) + * Likewise for SimpleConvertColumn.simpleConvert() (getConvert()) + */ + @Transient + default Function getFormatFn() + { + return getDefaultFormatFn(getName(), getJavaObjectClass(), getDisplayUnit(), getFormat(), null); + } + + @Transient + default Function getTsvFormatFn() + { + return getDefaultFormatFn(getName(), getJavaObjectClass(), getDisplayUnit(), getTsvFormatString(), DisplayColumn.tsvFormatSymbols); + } + + @Transient + default Function getConvertFn() + { + return getDefaultConvertFn(this); + } + + static Function getDefaultFormatFn(String colName, Class javaObjectClass, final Unit displayUnit, String formatString, DecimalFormatSymbols dfs) + { + final var format = null==formatString ? null : DisplayColumn.createFormat(formatString, javaObjectClass, dfs); + + if (null == format && null == displayUnit) + { + return (value) -> null==value ? "" : value instanceof String ? (String)value : ConvertUtils.convert(value); + } + + return (value) -> + { + if (null == value) + return ""; + + @NotNull String formattedString; + if (null != displayUnit && value instanceof Number) + { + Quantity q = (value instanceof Quantity) ? + (Quantity) value : + displayUnit.getKindOfQuantity().toQuantity((Number) value); + var doubleValue = q.doubleValue(displayUnit); + if (null == format) + formattedString = ConvertUtils.convert(doubleValue); + else + formattedString = format.format(doubleValue); + } + else if (null != format) + { + try + { + formattedString = format.format(value); + } + catch (IllegalArgumentException e) + { + LogHelper.getLogger(ColumnRenderProperties.class, "Column metadata").warn("Unable to apply format to {} value \"{}\" for column \"{}\", likely a SQL type mismatch between XML metadata and actual ResultSet", value.getClass().getName(), value, colName); + formattedString = ConvertUtils.convert(value); + } + } + else if (value instanceof String) + { + formattedString = (String) value; + } + else + { + formattedString = ConvertUtils.convert(value); + } + + return formattedString; + }; + } + + /* empty string -> null */ + static Function getDefaultConvertFn(ColumnRenderProperties col) + { + final Class javaClass = col.getJavaObjectClass(); + final var defaultUnit = col.getDisplayUnit(); + final @NotNull var jdbcType = col.getJdbcType(); + + if (null == defaultUnit) + { + return (value) -> + { + // quick check for unnecessary conversion + if (value == null || javaClass == value.getClass()) + return value; + if (value instanceof CharSequence) + ConvertUtils.convert(value.toString(), javaClass); + return jdbcType.convert(value); + }; + } + return defaultUnit::convert; + } } diff --git a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java index a8887f6f4fc..85d452d90a6 100644 --- a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java +++ b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java @@ -25,6 +25,7 @@ import org.labkey.api.gwt.client.DefaultScaleType; import org.labkey.api.gwt.client.DefaultValueType; import org.labkey.api.gwt.client.FacetingBehaviorType; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.FieldKey; import org.labkey.api.util.StringExpression; @@ -785,11 +786,27 @@ public final Class getJavaClass() @Override public Class getJavaClass(boolean isNullable) { + Class ret; + boolean isNumeric; PropertyType pt = getPropertyType(); if (pt != null) - return pt.getJavaType(); + { + ret = pt.getJavaType(); + isNumeric = pt.getJdbcType().isNumeric(); + } + else + { + ret = getJdbcType().getJavaClass(isNullable); + isNumeric = getJdbcType().isNumeric(); + } - return getJdbcType().getJavaClass(isNullable); + if (isNumeric) + { + Unit unit = getDisplayUnit(); + if (null != unit) + return unit.getQuantityClass(); + } + return ret; } @Override diff --git a/api/src/org/labkey/api/data/DataColumn.java b/api/src/org/labkey/api/data/DataColumn.java index f2470c9e5bc..e19c61ec6ef 100644 --- a/api/src/org/labkey/api/data/DataColumn.java +++ b/api/src/org/labkey/api/data/DataColumn.java @@ -28,6 +28,8 @@ import org.labkey.api.exp.property.PropertyService; import org.labkey.api.gwt.client.DefaultValueType; import org.labkey.api.gwt.client.model.PropertyValidatorType; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.DetailsURL; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryParseException; @@ -569,7 +571,7 @@ public HtmlString getFormattedHtml(RenderContext ctx) } else { - String formatted = formatValue(ctx, value, getTextExpressionCompiled(ctx), getFormat()); + String formatted = formatValue(ctx, value, getTextExpressionCompiled(ctx), getFormat(), getDisplayUnit()); if (getRequiresHtmlFiltering()) formatted = PageFlowUtil.filter(formatted); @@ -623,12 +625,18 @@ protected String getSelectInputDisplayValue(NamedObject entry) return entry.getObject().toString(); } - protected String getStringValue(Object value, boolean disabledInput) + protected String getStringValue(Object value, Unit unit, boolean disabledInput) { String strVal = ""; //UNDONE: Should use output format here. if (null != value) { + if (unit != null && value instanceof Number num) + { + Quantity quantity = (value instanceof Quantity q) ? q : unit.getKindOfQuantity().toQuantity(num); + value = quantity.value(unit); + } + // 4934: Don't render form input values with formatter since we don't parse formatted inputs on post. // For now, we can at least render disabled inputs with formatting since a // hidden input with the actual value is emitted for disabled items. @@ -657,7 +665,7 @@ public void renderInputHtml(RenderContext ctx, HtmlWriter out, Object value) boolean disabledInput = isDisabledInput(ctx); final String formFieldName = getFormFieldName(ctx); - String strVal = getStringValue(value, disabledInput); + String strVal = getStringValue(value, _boundColumn.getDisplayUnit(), disabledInput); if (_boundColumn.isAutoIncrement()) { @@ -940,7 +948,9 @@ public String getSortHandler(RenderContext ctx, Sort.SortDirection sort) // TODO: Treat null and empty the same instead? if (_caption == null) return null; - String title = _caption.eval(ctx); + var title = _caption.eval(ctx); + if (null != _displayColumn && null != _displayColumn.getDisplayUnit() && !StringUtils.isEmpty(_displayColumn.getDisplayUnit().toString())) + title += " (" + _displayColumn.getDisplayUnit() + ")"; return title.isEmpty() ? HtmlString.NBSP : HtmlString.of(title); } diff --git a/api/src/org/labkey/api/data/DisplayColumn.java b/api/src/org/labkey/api/data/DisplayColumn.java index a71f807180b..9a582246844 100644 --- a/api/src/org/labkey/api/data/DisplayColumn.java +++ b/api/src/org/labkey/api/data/DisplayColumn.java @@ -29,6 +29,8 @@ import org.labkey.api.compliance.PhiTransformedColumnInfo; import org.labkey.api.ontology.Concept; import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.FieldKey; import org.labkey.api.stats.ColumnAnalyticsProvider; import org.labkey.api.util.DOM; @@ -133,7 +135,9 @@ public int getRowSpan(RenderContext ctx) } @Override - public void addQueryColumns(Set fieldKeys) {} + public void addQueryColumns(Set fieldKeys) + { + } @Override public boolean shouldRenderInCurrentRow(RenderContext ctx) @@ -200,7 +204,8 @@ public StringExpression getURLExpression() return _urlExpression; } - public boolean includeURL() { + public boolean includeURL() + { return _urlExpression != null || _url != null; } @@ -257,7 +262,7 @@ public String renderURLTitle(RenderContext ctx) return _analyticsProviders; } - protected void addAnalyticsProvider(@NotNull ColumnAnalyticsProvider analyticsProvider) + protected void addAnalyticsProvider(@NotNull ColumnAnalyticsProvider analyticsProvider) { _analyticsProviders.add(analyticsProvider); } @@ -366,7 +371,8 @@ public void setFormatString(String formatString) // java 7 changed to using infinity symbols for formatting, which is challenging for tsv import/export // use old school "Infinity" for now - static DecimalFormatSymbols tsvFormatSymbols = new DecimalFormatSymbols(); + static public DecimalFormatSymbols tsvFormatSymbols = new DecimalFormatSymbols(); + static { tsvFormatSymbols.setInfinity(String.valueOf(Double.POSITIVE_INFINITY)); @@ -382,13 +388,15 @@ public void setTsvFormatString(String formatString) _tsvFormat = createFormat(formatString, tsvFormatSymbols); } + private Format createFormat(@Nullable String formatString, @Nullable DecimalFormatSymbols dfs) + { + return createFormat(formatString, getDisplayValueClass(), dfs); + } - private Format createFormat(String formatString, @Nullable DecimalFormatSymbols dfs) + public static Format createFormat(@Nullable String formatString, Class valueClass, @Nullable DecimalFormatSymbols dfs) { if (null != formatString) { - Class valueClass = getDisplayValueClass(); - try { if (Boolean.class.isAssignableFrom(valueClass) || boolean.class.isAssignableFrom(valueClass)) @@ -461,14 +469,14 @@ public StringExpression getTextExpressionCompiled(RenderContext ctx) public HtmlString getFormattedHtml(RenderContext ctx) { Format format = getFormat(); - return HtmlString.of(formatValue(ctx, getDisplayValue(ctx), getTextExpressionCompiled(ctx), format)); + Unit unit = getDisplayUnit(); + return HtmlString.of(formatValue(ctx, getDisplayValue(ctx), getTextExpressionCompiled(ctx), format, unit)); } /** * Format the display value as text only if there is a text expression or format configured for * the display column (which includes any project date and number format settings), * otherwise return null. - *

* No HTML encoding should be performed * @see #getFormattedHtml(RenderContext) */ @@ -506,31 +514,65 @@ public String getFormattedText(RenderContext ctx) * any html encoding. */ @NotNull - protected final String formatValue(RenderContext ctx, Object value, StringExpression expr, Format format) + protected final String formatValue(RenderContext ctx, final Object value, StringExpression expr, @Nullable Format format, @Nullable Unit displayUnit) { if (null == value) return ""; if (null != expr && expr.canRender(ctx.getFieldMap().keySet())) { + // TODO handle Quantity values here? return expr.eval(ctx); } + + @NotNull String formattedString; + if (null != displayUnit && value instanceof Number) + { + Quantity q = (value instanceof Quantity) ? + (Quantity)value : + displayUnit.getKindOfQuantity().toQuantity((Number)value); + /* DISPLAY WITH UNIT SUFFIX + if (null == format) + formattedString = q.format(displayUnit); + else + formattedString = q.format(displayUnit, format); + */ + /* DISPLAY WITHOUT UNIT SUFFIX */ + var doubleValue = q.doubleValue(displayUnit); + if (null == format) + formattedString = ConvertUtils.convert(doubleValue); + else + formattedString = format.format(doubleValue); + } else if (null != format) { try { - return format.format(value); + formattedString = format.format(value); } catch (IllegalArgumentException e) { LOG.warn("Unable to apply format to {} value \"{}\" for column \"{}\", likely a SQL type mismatch between XML metadata and actual ResultSet", value.getClass().getName(), value, getName()); - return ConvertUtils.convert(value); + formattedString = ConvertUtils.convert(value); } } else if (value instanceof String) - return (String)value; + { + formattedString = (String) value; + } + else + { + formattedString = ConvertUtils.convert(value); + } + + return formattedString; + } - return ConvertUtils.convert(value); + + Unit getDisplayUnit() + { + ColumnInfo col = getDisplayColumnInfo(); + return null != col ? col.getDisplayUnit() : null; } @@ -542,7 +584,8 @@ public String getTsvFormattedValue(RenderContext ctx) { format = getFormat(); } - return formatValue(ctx, getExportCompatibleValue(ctx), getTextExpressionCompiled(ctx), format); + Unit unit = getDisplayUnit(); + return formatValue(ctx, getExportCompatibleValue(ctx), getTextExpressionCompiled(ctx), format, unit); } public Object getExcelCompatibleValue(RenderContext ctx) @@ -550,7 +593,7 @@ public Object getExcelCompatibleValue(RenderContext ctx) return getExportCompatibleValue(ctx); } - public Object getExportCompatibleValue(RenderContext ctx) + public Object getExportCompatibleValue(RenderContext ctx) { Object value = getDisplayValue(ctx); if (null == value) diff --git a/api/src/org/labkey/api/data/Parameter.java b/api/src/org/labkey/api/data/Parameter.java index 1655ee18f7d..7f4dfe40c0e 100644 --- a/api/src/org/labkey/api/data/Parameter.java +++ b/api/src/org/labkey/api/data/Parameter.java @@ -23,6 +23,7 @@ import org.json.JSONObject; import org.labkey.api.attachments.AttachmentFile; import org.labkey.api.exp.Lsid; +import org.labkey.api.ontology.Quantity; import org.labkey.api.query.QueryService; import org.labkey.api.util.ResultSetUtil; import org.labkey.api.util.UnexpectedException; @@ -378,6 +379,9 @@ public static Object getValueToBind(@Nullable Object value, @Nullable JdbcType t if (value == null) return null; + if (value instanceof Quantity q) + value = q.value(); + if (value instanceof Double) return ResultSetUtil.mapJavaDoubleToDatabaseDouble(((Double)value)); if (value instanceof Number || value instanceof String) diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index fc09b222495..ea606b08129 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -22,6 +22,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; import org.labkey.api.query.AliasManager; import org.labkey.api.query.FieldKey; import org.labkey.api.settings.AppProps; @@ -29,6 +30,8 @@ import org.labkey.api.util.JdbcUtil; import org.labkey.api.util.Pair; +import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; @@ -518,10 +521,17 @@ public SQLFragment appendValue(Integer I) { if (null == I) return appendNull(); - getStringBuilder().append((int)I); + getStringBuilder().append(I.intValue()); return this; } + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + public SQLFragment appendValue(Long L) { if (null == L) @@ -530,11 +540,52 @@ public SQLFragment appendValue(Long L) return this; } + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + public SQLFragment appendValue(Float F) { if (null == F) return appendNull(); - getStringBuilder().append((float)F); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } return this; } @@ -542,8 +593,24 @@ public SQLFragment appendValue(Number N) { if (null == N) return appendNull(); - // Do we know that default java toString() for all numbers creates a valid SQL literal? - getStringBuilder().append(String.valueOf(N)); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(String.valueOf(N)); + } + else if (Double.isFinite(N.doubleValue())) + { + // Do we know that default java toString() for all numbers creates a valid SQL literal? + getStringBuilder().append(String.valueOf(N)); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } return this; } @@ -1283,10 +1350,9 @@ public static SQLFragment join(Iterable fragments, String separator /* REMOVE THIS - These methods are going away, but this allows us to merge w/o doing 100 modules at the same time */ - @Deprecated public SQLFragment append(@NotNull Container c) {return appendValue(c);} +// @Deprecated public SQLFragment append(@NotNull Container c) {return appendValue(c);} @Deprecated public SQLFragment append(Integer i) {return appendValue(i);} - @Deprecated public SQLFragment append(java.util.Date date) {return appendValue(date);} -// @Deprecated public SQLFragment append(Object o) {return append(String.valueOf(o));} +// @Deprecated public SQLFragment append(java.util.Date date) {return appendValue(date);} @Deprecated public SQLFragment appendStringLiteral(CharSequence s) {return appendValue(s);} /* END OF REMOVE THIS */ } diff --git a/api/src/org/labkey/api/data/TableViewForm.java b/api/src/org/labkey/api/data/TableViewForm.java index 94992b8d419..767f921f1f9 100644 --- a/api/src/org/labkey/api/data/TableViewForm.java +++ b/api/src/org/labkey/api/data/TableViewForm.java @@ -34,6 +34,7 @@ import org.labkey.api.action.NullSafeBindException; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.ontology.Quantity; import org.labkey.api.query.SchemaKey; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; @@ -409,33 +410,36 @@ protected void _populateValues(BindException errors) for (String propName : keys) { + ColumnInfo col = getColumnByFormFieldName(propName); String str = _stringValues.get(propName); String caption = _dynaClass.getPropertyCaption(propName); + Class propType = null; if (StringUtils.isEmpty(str)) str = null; - Class propType = null; - try { + if (null != str) { - propType = _dynaClass.getTruePropType(propName); - if (propType != null) + Object val; + if (null != col && null != col.getKindOfQuantity()) { - Object val = ConvertUtils.convert(str, propType); - values.put(propName, val); + val = Quantity.convert(str, col.getDisplayUnit()); } else { - values.put(propName, str); + propType = _dynaClass.getTruePropType(propName); + if (propType != null) + val = ConvertUtils.convert(str, propType); + else + val = str; } + values.put(propName, val); } else if (_validateRequired && null != _tinfo) { - ColumnInfo col = getColumnByFormFieldName(propName); - if (null == col || !col.isRequired()) { values.put(propName, null); @@ -472,7 +476,6 @@ else if (_validateRequired && null != _tinfo) boolean skipError = false; // Attempt to resolve lookups by display value - ColumnInfo col = getColumnByFormFieldName(propName); String defaultMessage = null; if (col != null && col.getFk() != null && col.getFk().allowImportByAlternateKey()) { diff --git a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java index b000c828be7..51a3e18ff5d 100644 --- a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java +++ b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java @@ -58,6 +58,7 @@ import org.labkey.api.exp.PropertyType; import org.labkey.api.files.FileContentService; import org.labkey.api.gwt.client.model.PropertyValidatorType; +import org.labkey.api.ontology.Unit; import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; @@ -491,7 +492,7 @@ protected class DerivationScopedConvertColumn extends SimpleConvertColumn public DerivationScopedConvertColumn(int index, SimpleConvertColumn convertCol, int derivationDataColInd, boolean isDerivation, @Nullable String presentDerivationWarning, @Nullable String presentNonDerivationWarning) { - super(convertCol.fieldName, convertCol.index, convertCol.type); + super(convertCol.fieldName, convertCol.index, convertCol.type, convertCol.defaultUnit); _convertCol = convertCol; this.index = index; this.derivationDataColInd = derivationDataColInd; @@ -568,12 +569,12 @@ protected class AliasColumn extends SimpleConvertColumn { AliasColumn(String fieldName, int index) { - super(fieldName, index, null); + super(fieldName, index, null, null); } AliasColumn(String fieldName, int index, JdbcType convert) { - super(fieldName, index, convert); + super(fieldName, index, convert, null); } @Override @@ -620,23 +621,22 @@ protected class SimpleConvertColumn implements Supplier { final int index; final @Nullable JdbcType type; + final @Nullable Unit defaultUnit; final String fieldName; final boolean _preserveEmptyString; - SimpleConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to) + SimpleConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to, @Nullable Unit defaultUnit) { - this.fieldName = fieldName; - this.index = indexFrom; - this.type = to; - _preserveEmptyString = false; + this(fieldName, indexFrom, to, defaultUnit, false); } - public SimpleConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to, boolean preserveEmptyString) + public SimpleConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to, @Nullable Unit defaultUnit, boolean preserveEmptyString) { this.index = indexFrom; this.type = to; this.fieldName = fieldName; - _preserveEmptyString = preserveEmptyString; + this.defaultUnit = (null==type || type.isNumeric()) ? defaultUnit : null; + _preserveEmptyString = preserveEmptyString && null != type && type.isText(); } @Override @@ -656,11 +656,20 @@ final public Object get() } } - protected Object convert(Object o) + protected Object simpleConvert(Object o) { - if (o instanceof String && JdbcType.VARCHAR.equals(type) && "".equals(o) && _preserveEmptyString) + if (_preserveEmptyString && "".equals(o)) return ""; - return null==type ? o : type.convert(o); + if (null != defaultUnit) + return defaultUnit.convert(o); + if (null != type) + return type.convert(o); + return o; + } + + protected Object convert(Object o) + { + return simpleConvert(o); } protected Object getSourceValue() @@ -784,15 +793,10 @@ private class MissingValueConvertColumn extends SimpleConvertColumn boolean supportsMissingValue = true; int indicator; - MissingValueConvertColumn(String fieldName, int index,JdbcType to) - { - super(fieldName, index, to); - indicator = NO_MV_INDEX; - } - MissingValueConvertColumn(String fieldName, int index, int indexIndicator, @Nullable JdbcType to) + MissingValueConvertColumn(String fieldName, int index, int indexIndicator, @Nullable JdbcType to, @Nullable Unit defaultUnit) { - super(fieldName, index, to); + super(fieldName, index, to, defaultUnit); indicator = indexIndicator; } @@ -841,9 +845,7 @@ public Object convert(Object value) Object innerConvert(Object value) { - if (type != null) - return type.convert(value); - return value; + return super.simpleConvert(value); } } @@ -852,10 +854,9 @@ private class PropertyConvertColumn extends MissingValueConvertColumn { @Nullable PropertyType pt; - PropertyConvertColumn(String fieldName, int fromIndex, int mvIndex, boolean supportsMissingValue, @Nullable PropertyType pt, @Nullable JdbcType type) + PropertyConvertColumn(String fieldName, int fromIndex, int mvIndex, boolean supportsMissingValue, @Nullable PropertyType pt, @Nullable JdbcType type, Unit defaultUnit) { - super(fieldName, fromIndex, mvIndex, type != null ? type : pt != null ? pt.getJdbcType() : null); - this.pt = pt; + super(fieldName, fromIndex, mvIndex, type != null ? type : pt != null ? pt.getJdbcType() : null, defaultUnit); this.supportsMissingValue = supportsMissingValue; } @@ -872,9 +873,9 @@ private class PropertyConvertAndTrimColumn extends PropertyConvertColumn { boolean trimRightOnly; - PropertyConvertAndTrimColumn(String fieldName, int fromIndex, int mvIndex, boolean supportsMissingValue, @Nullable PropertyType pt, @Nullable JdbcType type, boolean trimRightOnly) + PropertyConvertAndTrimColumn(String fieldName, int fromIndex, int mvIndex, boolean supportsMissingValue, @Nullable PropertyType pt, @Nullable JdbcType type, @Nullable Unit defaultUnit, boolean trimRightOnly) { - super(fieldName, fromIndex, mvIndex, supportsMissingValue, pt, type); + super(fieldName, fromIndex, mvIndex, supportsMissingValue, pt, type, defaultUnit); this.trimRightOnly = trimRightOnly; } @@ -900,7 +901,7 @@ private class MultiValueConvertColumn extends SimpleConvertColumn MultiValueConvertColumn(SimpleConvertColumn c) { - super(c.fieldName, c.index, c.type); + super(c.fieldName, c.index, c.type, c.defaultUnit); _c = c; } @@ -957,7 +958,7 @@ protected class RemappingConvertColumn extends SimpleConvertColumn public RemappingConvertColumn(final @NotNull SimpleConvertColumn convertCol, final int fromIndex, final @NotNull ColumnInfo toCol, RemapMissingBehavior missing, boolean includeTitleColumn, @NotNull LookupResolutionType lookupResolutionType) { - super(convertCol.fieldName, convertCol.index, convertCol.type); + super(convertCol.fieldName, convertCol.index, convertCol.type, convertCol.defaultUnit); _convertCol = convertCol; _toCol = toCol; _missing = missing; @@ -1350,9 +1351,9 @@ private SimpleConvertColumn createConvertColumn(@NotNull ColumnInfo col, int fro SimpleConvertColumn c; if (PropertyType.STRING == pt && (trimString || trimStringRight)) - c = new PropertyConvertAndTrimColumn(name, fromIndex, mvIndex, mv, pt, type, !trimString); + c = new PropertyConvertAndTrimColumn(name, fromIndex, mvIndex, mv, pt, type, col.getDisplayUnit(), !trimString); else - c = new PropertyConvertColumn(name, fromIndex, mvIndex, mv, pt, type); + c = new PropertyConvertColumn(name, fromIndex, mvIndex, mv, pt, type, col.getDisplayUnit()); ForeignKey fk = col.getFk(); LookupResolutionType lookupResolutionType = _context.getLookupResolutionType(); diff --git a/api/src/org/labkey/api/ontology/KindOfQuantity.java b/api/src/org/labkey/api/ontology/KindOfQuantity.java new file mode 100644 index 00000000000..3187026c2d5 --- /dev/null +++ b/api/src/org/labkey/api/ontology/KindOfQuantity.java @@ -0,0 +1,114 @@ +package org.labkey.api.ontology; + +import org.jetbrains.annotations.NotNull; + +import java.util.List; + + +/** + * These helpers follow UCUM conventions (or at least intend to). + *
+ * NOTE: We have an initialization ordering issue as Unit and KindOfQuantity are both enums and they point to each other. + * Just in case you were wondering why I don't use Unit directly in the constructor. + */ + +public enum KindOfQuantity +{ + Volume("volume", "ml") + { + @Override + List getCommonUnits() + { + return List.of(Unit.l, Unit.ml, Unit.ul); + } + }, + + Mass("mass", "g") + { + @Override + List getCommonUnits() + { + return List.of(Unit.kg, Unit.g, Unit.mg ,Unit.ug); + } + }, + + // Not a real unit per UCUM, but useful for annotation of "storage amount" for instance. + Count("", "unit") + { + @Override + List getCommonUnits() + { + return List.of(Unit.count, Unit.unit); + } + }; + + final @NotNull String name; + final @NotNull String storageUnitName; + Unit storageUnit = null; + + KindOfQuantity(@NotNull String name, @NotNull String storageUnitName) // , List list) + { + this.name = name; + this.storageUnitName = storageUnitName; + } + + @NotNull String getName() + { + return name; + } + + /* default unit for data entry */ + @NotNull Unit getDefaultDisplayUnit() + { + return storageUnit; + } + + /* unit used for database storage and in-memory representation of Quantity*/ + Unit getStorageUnit() + { + if (null == storageUnit) + storageUnit = Unit.fromName(storageUnitName); + return storageUnit; + } + + public Quantity toQuantity(Number n) + { + return Quantity.of(n, getStorageUnit()); + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + boolean accept(Unit unit) + { + return getStorageUnit().base == unit.base; + } + + abstract List getCommonUnits(); + + static KindOfQuantity getKindOfQuantity(String name) + { + if ("volume".equalsIgnoreCase(name)) + return Volume; + if ("vol".equalsIgnoreCase(name)) + return Volume; + if ("mass".equalsIgnoreCase(name)) + return Mass; + if ("weight".equalsIgnoreCase(name)) + return Mass; + if ("count".equalsIgnoreCase(name)) + return Count; + return null; + } + + // other potentially useful KindOfQuantity + // + // Fixed Unit e.g. arbitrary UCUM unit w/no conversion supported (basically a column annotation) + // if we support Fixed Unit, then maybe KindOfQuantity is used sparingly + // + // * time (aka duration, not datetime): 60s + // * length: 1m + // * area 1m2 + // * temperature/change in temperature has pitfalls + // * mass concentration + // * per volume (e.g. count per volume) /ml +} + diff --git a/api/src/org/labkey/api/ontology/Quantity.java b/api/src/org/labkey/api/ontology/Quantity.java new file mode 100644 index 00000000000..f68f87ddece --- /dev/null +++ b/api/src/org/labkey/api/ontology/Quantity.java @@ -0,0 +1,507 @@ +package org.labkey.api.ontology; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.util.Formats; + +import java.math.BigDecimal; +import java.text.Format; +import java.util.regex.Pattern; + +/* CONSIDER: it's tempting to store BigDecimal in memory after parse for math/conversion purposed, even if we store as double in the database */ + +/* + * There is a design decision here + * "value" is always relative to the Kind.getDefaultUnit() + * or + * "value" is relative to a stored/associated unit + * + * I think there is less room for error if Quantity is always the stored/database value, but + * this means that formatting code needs to be careful. + * + * There is also value in mimic-ing LabKey SQL semantics. I think there is value in NOT + * translating the "number". E.g. always have "Volume" typed data in the same scale, and always having Mass + * typed data in the same scale. Then units become a parse/display thing. + * + * LabKey SQL ideas + * + * CAST takes a bare number and creates a quantity type. Warns when converting a Quantity to another kind of Quantity + * + * 1.23 // untyped + * SET CAST(1.23 AS Mass) // 1.23g -- Convert to a Mass quantity with default unit g, display unit is 'g' + * CAST(1.23 AS Mass('g')); // 1.23g -- same as above, but 'g' is explicit + * CAST(1.23 AS Mass('kg')); // 1230g -- value is converted to g, display unit is 'kg' + * + * Since value is always stored in the same unit, there are no converting arithmetic methods/operators. + * LabKey SQL does not unit arithmetic. It will preserve the kind of quantity for simple calculations + * + * SUM(Mass) -> Mass + * Mass plus/minus Mass -> Mass + * Mass times/divide number -> Mass + * Mass/Volume = Double + * + * If you want to display a column in kg, don't do this "SELECT weight/1000". That will create a new small Mass quantity + * e.g. 7123g/1000 -> 7.123g. + * + * SELECT weight as weight_in_kg @unit=kg, + * or + * SELECT (CAST weight as DOUBLE)/1000.0 as weight_in_kg + * + */ +public class Quantity extends Number implements Comparable +{ + public final @NotNull KindOfQuantity kind; + public final Number value; + private final boolean isDouble; + + private Quantity() + { + throw new IllegalStateException(); + } + + /* Returns a quantity = value*units + * of(1, Kg) -> 1000g + */ + public static Quantity of(Number value, Unit unit) + { + if (value instanceof Quantity q) + { + if (unit.kindOfQuantity != q.kind) + throw new ConversionException("Cannot convert " + q.format() + " to " + unit); + return q; + } + if (value instanceof BigDecimal bd) + return new Quantity(unit.kindOfQuantity, bd, unit); + if (value instanceof Long l && (l > Integer.MAX_VALUE || l < Integer.MIN_VALUE)) + return new Quantity(unit.kindOfQuantity, BigDecimal.valueOf(l), unit); + return new Quantity(unit.kindOfQuantity, value.doubleValue(), unit); + } + + protected Quantity(@NotNull KindOfQuantity kind, BigDecimal value) + { + this.kind = kind; + this.value = value; + this.isDouble = false; + } + + protected Quantity(@NotNull KindOfQuantity kind, BigDecimal value, Unit from) + { + this.kind = kind; + this.value = from.toStorageUnitValue(value); + this.isDouble = this.value instanceof Double; + assert isDouble || this.value instanceof BigDecimal; + } + + protected Quantity(@NotNull KindOfQuantity kind, Double value) + { + this.kind = kind; + this.value = value; + this.isDouble = true; + } + + protected Quantity(@NotNull KindOfQuantity kind, Double value, Unit from) + { + this.kind = kind; + this.value = from.toStorageUnitValue(value); + this.isDouble = this.value instanceof Double; + assert isDouble || this.value instanceof BigDecimal; + } + + public double doubleValue(Unit unit) + { + if (unit == kind.getStorageUnit()) + return value.doubleValue(); + else + return Unit.convert(value.doubleValue(), kind.getStorageUnit(), unit); + } + + public String format() + { + return format(kind.getDefaultDisplayUnit()); + } + + public String format(Unit unit) + { + return value(unit) + unit.print; + } + + public String format(Format format) + { + return format(kind.getDefaultDisplayUnit(), format); + } + + public String format(Unit unit, Format format) + { + return format.format(value(unit)) + unit.print; + } + + /** + * Quantity extends Numeric so we kinda expect .toString() be a number. + * However, it's really important that quantity == ConvertUtils.convert(quantity.toString(), Quantity.class). + */ + @Override + public String toString() + { + var ret = format(kind.getStorageUnit()); + assert this == ConvertUtils.convert(ret, this.getClass()); + return ret; + } + + @Override + public boolean equals(Object obj) + { + return obj instanceof Quantity other && 0 == compareTo(other); + } + + @Override + public int compareTo(@NotNull Quantity other) + { + if (this.kind != other.kind) + throw new IllegalArgumentException("Can't compare " + this.kind + " and " + other.kind); + if (this.isDouble == other.isDouble) + return ((Comparable)this.value).compareTo(other.value); + BigDecimal thisDec = this.isDouble ? BigDecimal.valueOf((Double) this.value) : (BigDecimal)this.value; + BigDecimal otherDec = other.isDouble ? BigDecimal.valueOf((Double) other.value) : (BigDecimal)other.value; + return thisDec.compareTo(otherDec); + } + + public Number value() + { + return value; + } + + public Number value(Unit unit) + { + return unit.fromStorageUnitValue(value); + } + + /* Quantity implement Number, so most can can just pass this along without knowing. */ + @Override + public int intValue() + { + return value.intValue(); + } + + @Override + public long longValue() + { + return value.longValue(); + } + + @Override + public float floatValue() + { + return value.floatValue(); + } + + @Override + public double doubleValue() + { + return value.doubleValue(); + } + + + /*** PARSE ***/ + + private static final String NUMBER_REGEX = "(?[+\\-]?(?\\d*([.]\\d*)?)(?:[Ee][+\\-]?\\d+)?)"; + + // UCUM units can be pretty complicated, but we only use a small subset for now + private static final String SIMPLE_UNIT_REGEX = "(?[a-zA-Zμℓ]+)"; + // private static final String COMPLEX_UNIT ="(?[\\[/a-zA-Z°μℓ][\\[\\]_./a-zA-Z0-9°μℓ])?"; // unit contains []_./azAZ09°μ + + private static final Pattern pattern = Pattern.compile("^" + NUMBER_REGEX + "\\s*" + SIMPLE_UNIT_REGEX + "?$"); + + + private static Quantity parse(@NotNull String s) throws ConversionException + { + return parse(s, Unit.unit); + } + + + /** The defaultUnit has two purposes 1) define the expected KindOfQuantity 2) select a Unit if it is not explicit in the source */ + private static Quantity parse(@NotNull String s, @NotNull Unit defaultUnit) throws ConversionException + { + // We could probably create a real lexer/parser here, but we only need to be able to parse units we support + // FIRST, check if there is a space + s = s.trim(); + var split = s.indexOf(' '); + String valuePart=null; + String unitPart=null; + if (split > 0) + { + valuePart = s.substring(0, split).trim(); + unitPart = s.substring(split+1).trim(); + } + else + { + var m = pattern.matcher(s); + if (m.matches()) + { + String digits = m.group("digits"); + // there should be digits before or after the "." (this check avoids needing a more complicated regex) + if (!".".equals(digits) && !"".equals(digits)) + { + valuePart = m.group("number"); + if (StringUtils.isNotBlank(m.group("unit"))) //m.namedGroups().containsKey("unit")) + unitPart = m.group("unit"); + } + } + } + + if (StringUtils.isBlank(valuePart)) + throw new ConversionException("Could not parse number"); + + try + { + Number value = StringUtils.containsAny(valuePart,"eE") ? + Double.valueOf(valuePart) : + new BigDecimal(valuePart); + var unit = StringUtils.isBlank(unitPart) ? defaultUnit : Unit.fromName(unitPart); + if (null == unit) + throw new ConversionException("Could not parse unit: " + unitPart); + if (!defaultUnit.kindOfQuantity.accept(unit)) + throw new ConversionException("Quantity is of wrong type: expected " + defaultUnit.kindOfQuantity.getName() + " found " + unit); + return Quantity.of(value, unit); + } + catch (IllegalArgumentException x) + { + throw new ConversionException("could not parse", x); + } + } + + + /*** CONVERSION **/ + + public abstract static class Mass_pg extends Quantity {} + public abstract static class Mass_ng extends Quantity {} + public abstract static class Mass_ug extends Quantity {} + public abstract static class Mass_mg extends Quantity {} + public abstract static class Mass_g extends Quantity {} + public abstract static class Mass_kg extends Quantity {} + public abstract static class Mass_Megag extends Quantity {} + + public abstract static class Volume_pl extends Quantity {} + public abstract static class Volume_nl extends Quantity {} + public abstract static class Volume_ul extends Quantity {} + public abstract static class Volume_ml extends Quantity {} + public abstract static class Volume_l extends Quantity {} + public abstract static class Volume_kl extends Quantity {} + public abstract static class Volume_Megal extends Quantity {} + + public abstract static class Volume extends Quantity + { + } + + + // convert (ala BeanUtils.Converter to Quantity + public static Quantity convert(Object o, Unit unit) + { + if (null == o) + return null; + if (o instanceof Quantity q) + { + if (q.kind == unit.kindOfQuantity) + return q; + throw new ConversionException("Cannot convert " + q.format() + " to " + unit); + } + if (o instanceof Number n) + { + return Quantity.of(n, unit); + } + return Quantity.parse(String.valueOf(o), unit); + } + + + public static Converter converterFor(Unit unit) + { + return new Converter() + { + @Override + public T convert(Class aClass, Object o) + { + return (T)Quantity.convert(o, unit); + } + }; + } + + static void registerQuantityConverters() + { + for (Unit unit : Unit.values()) + ConvertUtils.register(converterFor(unit), unit.getQuantityClass()); + } + + static + { + registerQuantityConverters(); + } + + + /*** TEST ***/ + + public static class TestCase extends Assert + { + void failToParse(String s) + { + try + { + parse(s); + fail("expected exception for " + s); + } + catch (ConversionException x) + { + // YEAH + } + } + + void failToParse(String s, Unit defaultUnit) + { + try + { + parse(s, defaultUnit); + fail("expected exception for " + s); + } + catch (ConversionException x) + { + // YEAH + } + } + + @Test + public void testParse() + { + assertEquals(Quantity.of(1, Unit.g), parse("1", Unit.g)); + assertEquals(Quantity.of(1, Unit.g), parse("1g", Unit.g)); + assertEquals(Quantity.of(1, Unit.g), parse("0.001kg", Unit.g)); + + assertEquals(Quantity.of(1, Unit.count), parse("1")); + assertEquals(Quantity.of(1, Unit.count), parse("1 unit")); + assertEquals(Quantity.of(0, Unit.count), parse("0 units")); + assertEquals(Quantity.of(0, Unit.count), parse("0count")); + + assertEquals(parse("1000mg", Unit.g), parse("0.001kg", Unit.g)); + assertEquals(parse(" 1000mg", Unit.g), parse("0.001kg", Unit.g)); + assertEquals(parse("1000mg ", Unit.g), parse("0.001kg", Unit.g)); + assertEquals(parse("1000 mg", Unit.g), parse("0.001kg", Unit.g)); + assertEquals(parse("1000 mg", Unit.g), parse("0.001kg", Unit.g)); + } + + @Test + public void testFailToParseNoDigit() + { + failToParse("kg"); + failToParse("test"); + failToParse("+"); + failToParse("."); + failToParse("-"); + failToParse("+e1"); + } + + @Test + public void testFailToParseInvalidUnit() + { + failToParse("124xyz"); + failToParse("124 xyz"); + failToParse("g100"); + } + + @Test + public void testFailToParseCantConvert() + { + failToParse("1g", Unit.l); + failToParse("1g", Unit.ml); + failToParse("1g", Unit.count); + failToParse("1g", Unit.unit); + } + + @Test + public void testPattern() + { + // no exponents + assertEquals(0.1, Double.valueOf(".1"), 0.0); + assertEquals(0.1, Double.valueOf("+.1"), 0.0); + assertEquals(-0.1, Double.valueOf("-.1"), 0.0); + assertTrue(pattern.matcher("1").matches()); + assertTrue(pattern.matcher("+1").matches()); + assertTrue(pattern.matcher("-1").matches()); + assertTrue(pattern.matcher("1.").matches()); + assertTrue(pattern.matcher("+1.").matches()); + assertTrue(pattern.matcher("-1.").matches()); + assertTrue(pattern.matcher(".2").matches()); + assertTrue(pattern.matcher("+.2").matches()); + assertTrue(pattern.matcher("-.2").matches()); + + // no digits + assertTrue(pattern.matcher("+").matches()); + assertTrue(pattern.matcher(".").matches()); + assertTrue(pattern.matcher("-").matches()); + assertTrue(pattern.matcher("+e1").matches()); + + //exponents + assertEquals(0.1e2, Double.valueOf(".1e2"), 0.0); + assertTrue(pattern.matcher(".1e2").matches()); + assertTrue(pattern.matcher("1.23e4").matches()); + assertTrue(pattern.matcher("1.23E4").matches()); + assertTrue(pattern.matcher("+1.23E4").matches()); + assertTrue(pattern.matcher("-1.23E-4").matches()); + + // units + assertTrue(pattern.matcher("1.23μF").matches()); + assertTrue(pattern.matcher("1.2e3 mℓ").matches()); + assertTrue(pattern.matcher("test").matches()); + assertTrue(pattern.matcher("g").matches()); + assertTrue(pattern.matcher("mℓ").matches()); + } + + + @Test + public void testConversion() + { + Quantity q; + registerQuantityConverters(); + + q = (Quantity)ConvertUtils.convert("1.234kg", Quantity.Mass_g.class); + assertEquals(Quantity.class, q.getClass()); + assertEquals(new Quantity(KindOfQuantity.Mass, 1234d), q); + + q = (Quantity)ConvertUtils.convert("1234", Quantity.Mass_g.class); + assertEquals(Quantity.class, q.getClass()); + assertEquals(new Quantity(KindOfQuantity.Mass, 1234d), q); + + q = (Quantity)ConvertUtils.convert("1234", Quantity.Mass_kg.class); + assertEquals(Quantity.class, q.getClass()); + assertEquals(new Quantity(KindOfQuantity.Mass, 1234000d), q); + } + + @Test + public void testDoubleValue() + { + Quantity q = Quantity.of(1, Unit.g); + assertEquals(1.0, q.doubleValue(Unit.g), 0.0); + assertEquals(0.001, q.doubleValue(Unit.kg), 0.0); + assertEquals(1000.0, q.doubleValue(Unit.mg), 0.001); + assertEquals(1000000.0, q.doubleValue(Unit.ug), 0.001); + assertEquals(1000000000.0, q.doubleValue(Unit.ng), 0.001); + } + + @Test + public void testFormat() + { + Quantity q = Quantity.of(1, Unit.g); + assertEquals("1.0g", q.format()); + assertEquals("1.0g", q.format(Unit.g)); + assertEquals("0.001kg", q.format(Unit.kg)); + assertEquals("1000.0mg", q.format(Unit.mg)); + + // with format + assertEquals("1.000g", q.format(Formats.f3)); + assertEquals("1.000g", q.format(Unit.g, Formats.f3)); + assertEquals("0.001kg", q.format(Unit.kg, Formats.f3)); + assertEquals("1000000ug", q.format(Unit.ug, Formats.fv3)); + } + } +} diff --git a/api/src/org/labkey/api/ontology/Unit.java b/api/src/org/labkey/api/ontology/Unit.java new file mode 100644 index 00000000000..c8caaa05699 --- /dev/null +++ b/api/src/org/labkey/api/ontology/Unit.java @@ -0,0 +1,285 @@ +package org.labkey.api.ontology; + +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.function.Function; + +public enum Unit +{ + unit(KindOfQuantity.Count, null, 1.0, "", + Quantity.class, + "unit", "units"), + count(KindOfQuantity.Count, unit, 1.0, "", + Quantity.class, + "count", "count"), + + ml(KindOfQuantity.Volume, null, 1e0, "ml", + Quantity.Volume_ml.class, + "milliliter", "milliliters", + "mL", "millilitre", "millilitres"), + // UCUM prefers "l", but "L" is also common and already supported by inventory (sorry Lambert) + l(KindOfQuantity.Volume, ml, 1e3, "l", + Quantity.Volume_l.class, + "liter", "liters", + "L", "ℓ", "litre", "liters"), + // is it better to include these little used units, to avoid future case-sensitivity problems? + Ml(KindOfQuantity.Volume, ml, 1e9, "Ml", + Quantity.Volume_Megal.class, + "megaliter", "megaliters", + "ML", "megalitre", "megalitres"), + kl(KindOfQuantity.Volume, ml, 1e6, "kl", + Quantity.Volume_kl.class, + "kiloliter", "kiloliters", + "kL", "kilolitre", "kilolitres"), + ul(KindOfQuantity.Volume, ml, 1e-3, "ul", + Quantity.Volume_ul.class, + "microliter", "microliters", + "uL", "μl", "μL", "microlitre", "microlitres"), + nl(KindOfQuantity.Volume, ml, 1e-6, "nl", + Quantity.Volume_nl.class, + "nanoliter", "nanoliters", + "nL", "nanolitre", "nanolitres"), + pl(KindOfQuantity.Volume, ml, 1e-9, "pl", + Quantity.Volume_pl.class, + "picoliter", "picoliters", + "pL", "picolitre", "picolitres"), + + g(KindOfQuantity.Mass, null, 1e0, "g", + Quantity.Mass_g.class, + "gram", "grams"), + Mg(KindOfQuantity.Mass, g, 1e6, "Mg", + Quantity.Mass_Megag.class, + "megagram", "megagrams", + "tonne", "tonnes"), + kg(KindOfQuantity.Mass, g, 1e3, "kg", + Quantity.Mass_kg.class, + "kilogram", "kilograms"), + mg(KindOfQuantity.Mass, g, 1e-3, "mg", + Quantity.Mass_mg.class, + "milligram", "milligrams"), + ug(KindOfQuantity.Mass, g, 1e-6, "ug", + Quantity.Mass_ug.class, + "microgram", "micrograms", + "μg"), + ng(KindOfQuantity.Mass, g, 1e-9, "ng", + Quantity.Mass_ng.class, + "nanogram", "nanograms"), + pg(KindOfQuantity.Mass, g, 1e-12, "pg", + Quantity.Mass_pg.class, + "picogram", "picograms"); + + + @Getter + final @NotNull KindOfQuantity kindOfQuantity; + @Getter + final @NotNull Unit base; + final @NotNull String print; + // this is not a 'real' class, but is used for ConvertHelper binding + @Getter + final @NotNull Class quantityClass; + final @NotNull String singular; + final @NotNull String plural; + final String[] otherNames; + final double value; + + Unit(KindOfQuantity kind, Unit base, double value, @NotNull String printName, + Class quantityClass, + @NotNull String singular, @NotNull String plural, String... otherNames) + { + this.kindOfQuantity = kind; + this.base = null == base ? this : base; + this.value = value; + this.print = printName; + this.quantityClass = quantityClass; + this.singular = singular; + this.plural = plural; + this.otherNames = null==otherNames || otherNames.length==0 ? null : otherNames; + } + + public boolean isBase() + { + return this == base; + } + + public boolean isCompatible(Unit other) + { + return other.base == base; + } + + public double toBaseUnitValue(double v) + { + return v * this.value; + } + + public double fromBaseUnitValue(double v) + { + return v / this.value; + } + + public Number toStorageUnitValue(Number v) + { + if (this == kindOfQuantity.getStorageUnit()) + return v; + return convert(v.doubleValue(), this, kindOfQuantity.storageUnit); + } + + public Number fromStorageUnitValue(Number v) + { + if (this == kindOfQuantity.getStorageUnit()) + return v; + return convert(v.doubleValue(), kindOfQuantity.storageUnit, this); + } + + @Override + public String toString() + { + return print; + } + + static final HashMap unitMap = new HashMap<>(Unit.values().length*10); + static + { + for (Unit unit : Unit.values()) + { + unitMap.put(unit.print, unit); + unitMap.put(unit.singular, unit); + unitMap.put(unit.plural, unit); + if (null != unit.otherNames) + for (String name : unit.otherNames) + unitMap.put(name, unit); + } + } + + + public static Unit fromName(String unitName) + { + if (StringUtils.isEmpty(unitName)) + return null; + Unit unit = unitMap.get(unitName); + if (null == unit) + unit = unitMap.get(unitName.toLowerCase()); + return unit; + } + + // don't assume multiplicative relation between units (e.g. Kelvin and Celsius) + static Function convertFn(Unit from, Unit to) + { + if (from == to) + return Function.identity(); + if (from.base != to.base) + throw new IllegalArgumentException("Can't convert " + from.name() + " to " + to.name()); + return (x) -> to.fromBaseUnitValue(from.toBaseUnitValue(x)); + } + + public static double convert(double value, Unit from, Unit to) + { + if (from.base != to.base) + throw new IllegalArgumentException("Can't convert " + from.name() + " to " + to.name()); + return from==to ? value : to.fromBaseUnitValue(from.toBaseUnitValue(value)); + } + + public Quantity convert(Object value) + { + return Quantity.convert(value, this); + } + + public static class TestCase extends Assert + { + @Test + public void testIsBase() + { + assertTrue(Unit.ml.isBase()); + assertFalse(Unit.l.isBase()); + assertTrue(Unit.g.isBase()); + assertFalse(Unit.kg.isBase()); + assertTrue(Unit.unit.isBase()); + assertFalse(Unit.count.isBase()); + } + + @Test + public void testIsCompatible() + { + assertTrue(Unit.ml.isCompatible(Unit.ul)); + assertTrue(Unit.ml.isCompatible(Unit.l)); + assertFalse(Unit.ml.isCompatible(Unit.g)); + assertTrue(Unit.g.isCompatible(Unit.mg)); + assertFalse(Unit.g.isCompatible(Unit.ml)); + assertTrue(Unit.unit.isCompatible(Unit.count)); + assertFalse(Unit.unit.isCompatible(Unit.ml)); + } + + @Test + public void testBaseUnitValue() + { + assertEquals(1e0, Unit.ml.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e3, Unit.l.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e-3, Unit.ul.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e0, Unit.g.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e-3, Unit.mg.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e-6, Unit.ug.toBaseUnitValue(1.0), 0.00001); + assertEquals(1e0, Unit.count.toBaseUnitValue(1.0), 0.00001); + } + + @Test + public void testFromBaseUnitValue() + { + assertEquals(1.0, Unit.ml.fromBaseUnitValue(1e0), 0.00001); + assertEquals(1.0, Unit.l.fromBaseUnitValue(1e3), 0.00001); + assertEquals(1.0, Unit.ul.fromBaseUnitValue(1e-3), 0.00001); + assertEquals(1.0, Unit.g.fromBaseUnitValue(1e0), 0.00001); + assertEquals(1.0, Unit.mg.fromBaseUnitValue(1e-3), 0.00001); + assertEquals(1.0, Unit.ug.fromBaseUnitValue(1e-6), 0.00001); + assertEquals(1.0, Unit.count.fromBaseUnitValue(1e0), 0.00001); + } + + @Test + public void testToStorageUnitValue() + { + assertEquals(1.0, Unit.ml.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1000.0, Unit.l.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(0.001, Unit.ul.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1.0, Unit.g.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(0.001, Unit.mg.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(0.000001, Unit.ug.toStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1.0, Unit.count.toStorageUnitValue(1.0).doubleValue(), 0.00001); + } + + @Test + public void testFromStorageUnitValue() + { + assertEquals(1.0, Unit.ml.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(0.001, Unit.l.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1000.0, Unit.ul.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1.0, Unit.g.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1000.0, Unit.mg.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1000000.0, Unit.ug.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + assertEquals(1.0, Unit.count.fromStorageUnitValue(1.0).doubleValue(), 0.00001); + } + + @Test + public void testFromName() + { + assertEquals(Unit.ml, Unit.fromName("ml")); + assertEquals(Unit.ml, Unit.fromName("mL")); + assertEquals(Unit.ml, Unit.fromName("milliliter")); + assertEquals(Unit.ml, Unit.fromName("milliliters")); + assertEquals(Unit.ml, Unit.fromName("millilitre")); + assertEquals(Unit.ml, Unit.fromName("millilitres")); + + assertEquals(Unit.l, Unit.fromName("l")); + assertEquals(Unit.l, Unit.fromName("L")); + assertEquals(Unit.l, Unit.fromName("liter")); + assertEquals(Unit.l, Unit.fromName("liters")); + assertEquals(Unit.l, Unit.fromName("litre")); + assertEquals(Unit.l, Unit.fromName("liters")); + + assertNull(Unit.fromName(null)); + assertNull(Unit.fromName("")); + } + } +} diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index 872719b2918..500c85e05d7 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -719,7 +719,7 @@ protected Map coerceTypes(Map row) if (PropertyType.FILE_LINK.equals(col.getPropertyType()) && value instanceof String strVal) value = ExpDataFileConverter.convert(strVal); else - value = ConvertUtils.convert(value.toString(), col.getJavaObjectClass()); + value = col.getConvertFn().apply(value); } catch (ConvertHelper.FileConversionException e) { diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index 4d4c06109a5..2e33d3bb2b7 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -837,23 +837,16 @@ protected Object convertColumnValue(ColumnInfo col, Object value, User user, Con // improve handling of conversion errors try { - switch (col.getJdbcType()) + if (PropertyType.FILE_LINK == col.getPropertyType()) { - case DATE, TIME, TIMESTAMP: - return value instanceof Date ? value : ConvertUtils.convert(value.toString(), Date.class); - default: - if (PropertyType.FILE_LINK == col.getPropertyType()) - { - if ((value instanceof MultipartFile || value instanceof AttachmentFile)) - { - FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); - value = fl.toNioPathForRead().toString(); - } - - return ExpDataFileConverter.convert(value); - } - return ConvertUtils.convert(value.toString(), col.getJdbcType().getJavaClass()); + if ((value instanceof MultipartFile || value instanceof AttachmentFile)) + { + FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); + value = fl.toNioPathForRead().toString(); + } + return ExpDataFileConverter.convert(value); } + return col.getConvertFn().apply(value); } catch (ConversionException e) { diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index a4befe5bce7..9d0ef0270ab 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -49,6 +49,7 @@ public interface AppProps String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; String DEPRECATED_OBJECT_LEVEL_DISCUSSIONS = "deprecatedObjectLevelDiscussions"; String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; String UNKNOWN_VERSION = "Unknown Release Version"; diff --git a/api/src/org/labkey/api/view/TypeAheadSelectDisplayColumn.java b/api/src/org/labkey/api/view/TypeAheadSelectDisplayColumn.java index 9dd53bcbebb..db4321ef35f 100644 --- a/api/src/org/labkey/api/view/TypeAheadSelectDisplayColumn.java +++ b/api/src/org/labkey/api/view/TypeAheadSelectDisplayColumn.java @@ -22,6 +22,7 @@ import org.labkey.api.data.DisplayColumnFactory; import org.labkey.api.data.ForeignKey; import org.labkey.api.data.RenderContext; +import org.labkey.api.ontology.Unit; import org.labkey.api.util.JavaScriptFragment; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.UniqueID; @@ -52,7 +53,10 @@ public TypeAheadSelectDisplayColumn(ColumnInfo col, Integer maxRows) @Override public void renderInputHtml(RenderContext ctx, HtmlWriter out, Object value) { - ForeignKey fk = getBoundColumn().getFk(); + ColumnInfo boundColumn = getBoundColumn(); + ForeignKey fk = boundColumn.getFk(); + Unit unit = boundColumn.getDisplayUnit(); + // currently only supported for lookup columns with a defined schema/query if (fk == null) { @@ -62,7 +66,7 @@ public void renderInputHtml(RenderContext ctx, HtmlWriter out, Object value) String formFieldName = getFormFieldName(ctx); boolean disabledInput = isDisabledInput(ctx); - String strVal = getStringValue(value, disabledInput); + String strVal = getStringValue(value, unit, disabledInput); String renderId = "query-select-div-" + UniqueID.getRequestScopedUID(ctx.getRequest()); StringBuilder sb = new StringBuilder(); diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 5dfeff8027b..0f9e975f763 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -79,6 +79,8 @@ import org.labkey.api.module.SpringModule; import org.labkey.api.module.Summary; import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryService; @@ -256,8 +258,9 @@ protected void init() { OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); RoleManager.registerPermission(new DesignVocabularyPermission(), true); RoleManager.registerRole(new SampleTypeDesignerRole()); @@ -1033,7 +1036,9 @@ public Set getUnitTests() LSIDRelativizer.TestCase.class, Lsid.TestCase.class, LsidUtils.TestCase.class, - PropertyController.TestCase.class + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class ); } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 63b5a3a8678..c8c41ce1d8a 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1924,7 +1924,8 @@ protected class SampleUnitsConvertColumn extends SimpleTranslator.SimpleConvertC { public SampleUnitsConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to) { - super(fieldName, indexFrom, to, true); + // TODO reconcile unit handling + super(fieldName, indexFrom, to, null, true); } @Override @@ -1939,7 +1940,8 @@ protected class SampleAmountConvertColumn extends SimpleTranslator.SimpleConvert { public SampleAmountConvertColumn(String fieldName, int indexFrom, @Nullable JdbcType to) { - super(fieldName, indexFrom, to, true); + // TODO reconcile unit handling + super(fieldName, indexFrom, to, null, true); } @Override @@ -1955,7 +1957,8 @@ protected class AliquotRollupConvertColumn extends SimpleConvertColumn public AliquotRollupConvertColumn(String fieldName, @Nullable JdbcType to, int aliquotedFromColInd) { - super(fieldName, 0, to, true); + // TODO reconcile unit handling + super(fieldName, 0, to, null, true); this.aliquotedFromColInd = aliquotedFromColInd; } diff --git a/query/src/org/labkey/query/sql/QNode.java b/query/src/org/labkey/query/sql/QNode.java index d9af20d205b..4e61134765f 100644 --- a/query/src/org/labkey/query/sql/QNode.java +++ b/query/src/org/labkey/query/sql/QNode.java @@ -472,7 +472,7 @@ public void testType() test("CONVERT(now(),SQL_DATE)", JdbcType.DATE); test("CAST(now() AS DATE)", JdbcType.DATE); test("CASE WHEN 1=1 THEN CONVERT(now(),SQL_DATE) ELSE CAST(now() AS DATE) END", JdbcType.DATE); - test("1 + 2.0", JdbcType.DOUBLE); + test("1 + 2.0", JdbcType.DECIMAL); test("'hello' | 'world'", JdbcType.VARCHAR); } diff --git a/query/src/org/labkey/query/sql/QNumber.java b/query/src/org/labkey/query/sql/QNumber.java index 3876f7bcb50..712526779c0 100644 --- a/query/src/org/labkey/query/sql/QNumber.java +++ b/query/src/org/labkey/query/sql/QNumber.java @@ -33,7 +33,13 @@ public class QNumber extends QExpr implements IConstant public QNumber(String s) { - if (StringUtils.containsOnly(s,"0123456789")) + String substring = s; + if (s.startsWith("0x")) + setValue(convertInteger(s)); + else if (s.startsWith("+") || s.startsWith("-")) + substring = s.substring(1); + + if (StringUtils.containsOnly(substring,"0123456789")) setValue(convertInteger(s)); else setValue(convertDouble(s)); @@ -142,13 +148,37 @@ Number convertInteger(String s) Number convertDouble(String s) { - try + boolean floatish = false; + if (s.endsWith("f") || s.endsWith("F") || s.endsWith("d") || s.endsWith("D")) { - return Double.parseDouble(s); + s = s.substring(0,s.length()-1); + floatish = true; } - catch (NumberFormatException x) + if (StringUtils.containsAny(s, "eE")) + floatish = true; + if (floatish) + { + // try double first fall-back to decimal + try + { + return Double.parseDouble(s); + } + catch (NumberFormatException x) + { + return new BigDecimal(s); + } + } + else { - return new BigDecimal(s); + // try decimal first fall-back to double + try + { + return new BigDecimal(s); + } + catch (NumberFormatException x) + { + return Double.parseDouble(s); + } } } diff --git a/query/src/org/labkey/query/sql/QString.java b/query/src/org/labkey/query/sql/QString.java index 5b3df398b4a..961b2af4e45 100644 --- a/query/src/org/labkey/query/sql/QString.java +++ b/query/src/org/labkey/query/sql/QString.java @@ -50,7 +50,7 @@ public String getValue() @Override public void appendSql(SqlBuilder builder, Query query) { - builder.appendStringLiteral(getValue()); + builder.appendValue(getValue()); } @Override diff --git a/query/src/org/labkey/query/sql/SqlParser.java b/query/src/org/labkey/query/sql/SqlParser.java index 4a8b51f19ac..f8d70874a22 100644 --- a/query/src/org/labkey/query/sql/SqlParser.java +++ b/query/src/org/labkey/query/sql/SqlParser.java @@ -2046,8 +2046,8 @@ public static class SqlParserTestCase extends Assert new Pair<>("'this ' || 'that'", JdbcType.VARCHAR), new Pair<>("1 || ' plus ' || 2", JdbcType.VARCHAR), new Pair<>("1 + 2", JdbcType.INTEGER), - new Pair<>("1.0 + 2.1", JdbcType.DOUBLE), - new Pair<>("1 + 2.1", JdbcType.DOUBLE), + new Pair<>("1.0 + 2.1", JdbcType.DECIMAL), + new Pair<>("1 + 2.1", JdbcType.DECIMAL), new Pair<>("ROUND(0.0,1)", JdbcType.DOUBLE), new Pair<>("1 + ROUND(0.0,1)", JdbcType.DOUBLE), new Pair<>("CASE WHEN TRUE THEN ROUND(0.0,1) ELSE ROUND(0.0,1) END", JdbcType.DOUBLE)