From 5067059af398251f4aa60fc22b16b99e46806590 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 30 Mar 2022 10:10:48 +1000 Subject: [PATCH 01/83] Initial work on getting basic Quantity equality working --- fhir-server/pom.xml | 7 + .../pathling/fhirpath/element/CodingPath.java | 8 +- .../fhirpath/element/ElementDefinition.java | 1 + .../fhirpath/element/QuantityPath.java | 35 +- .../fhirpath/encoding/QuantityEncoding.java | 40 + .../fhirpath/literal/QuantityLiteralPath.java | 147 +- .../fhirpath/parser/LiteralTermVisitor.java | 36 +- .../java/au/csiro/pathling/spark/Spark.java | 7 +- .../{sql => terminology}/CodingToLiteral.java | 2 +- .../terminology/ucum/QuantityEquality.java | 79 + .../csiro/pathling/terminology/ucum/Ucum.java | 37 + .../src/main/resources/tx/ucum-essence.xml | 2110 +++++++++++++++++ .../pathling/test/UnitTestDependencies.java | 16 +- 13 files changed, 2491 insertions(+), 34 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java rename fhir-server/src/main/java/au/csiro/pathling/{sql => terminology}/CodingToLiteral.java (96%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java create mode 100644 fhir-server/src/main/resources/tx/ucum-essence.xml diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index fbe35f1bd6..6e91f6ae06 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -148,6 +148,13 @@ ${pathling.antlrVersion} + + + org.fhir + ucum + 1.0.3 + + com.amazonaws diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java index fcc9c9f00e..18003e7b1d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java @@ -16,7 +16,7 @@ import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; -import au.csiro.pathling.sql.CodingToLiteral; +import au.csiro.pathling.terminology.CodingToLiteral; import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.List; @@ -104,11 +104,9 @@ public static Optional valueFromRow(@Nonnull final Row row, final int co public static Function buildComparison(@Nonnull final Comparable source, @Nonnull final ComparisonOperation operation) { if (ComparisonOperation.EQUALS.equals(operation)) { - return Comparable - .buildComparison(source, codingEqual()); + return Comparable.buildComparison(source, codingEqual()); } else if (ComparisonOperation.NOT_EQUALS.equals(operation)) { - return Comparable - .buildComparison(source, codingNotEqual()); + return Comparable.buildComparison(source, codingNotEqual()); } else { throw new InvalidUserInputError( "Coding type does not support comparison operator: " + operation); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/ElementDefinition.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/ElementDefinition.java index 5d6eef2ff9..fcf22b777a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/ElementDefinition.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/ElementDefinition.java @@ -55,6 +55,7 @@ public class ElementDefinition { .put(FHIRDefinedType.TIME, TimePath.class) .put(FHIRDefinedType.CODING, CodingPath.class) .put(FHIRDefinedType.QUANTITY, QuantityPath.class) + .put(FHIRDefinedType.SIMPLEQUANTITY, QuantityPath.class) .put(FHIRDefinedType.REFERENCE, ReferencePath.class) .put(FHIRDefinedType.EXTENSION, ExtensionPath.class) .build(); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 6015caca44..f899c3bdbe 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -6,12 +6,20 @@ package au.csiro.pathling.fhirpath.element; +import au.csiro.pathling.errors.InvalidUserInputError; +import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.literal.NullLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.terminology.ucum.QuantityEquality; +import com.google.common.collect.ImmutableSet; import java.util.Optional; +import java.util.function.Function; import javax.annotation.Nonnull; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.apache.spark.sql.functions; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; /** @@ -19,7 +27,10 @@ * * @author John Grimes */ -public class QuantityPath extends ElementPath { +public class QuantityPath extends ElementPath implements Comparable { + + public static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet + .of(QuantityPath.class, QuantityLiteralPath.class, NullLiteralPath.class); protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Optional eidColumn, @@ -30,4 +41,26 @@ protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset< thisColumn, fhirType); } + @Nonnull + @Override + public Function getComparison(@Nonnull final ComparisonOperation operation) { + return buildComparison(this, operation); + } + + public static Function buildComparison(@Nonnull final Comparable source, + final @Nonnull ComparisonOperation operation) { + if (operation == ComparisonOperation.EQUALS) { + return Comparable.buildComparison(source, + (l, r) -> functions.callUDF(QuantityEquality.FUNCTION_NAME, l, r)); + } else { + throw new InvalidUserInputError( + "Quantity type does not support comparison operator: " + operation); + } + } + + @Override + public boolean isComparableTo(@Nonnull final Class type) { + return COMPARABLE_TYPES.contains(type); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java new file mode 100644 index 0000000000..00a32b3a0b --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.encoding; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; + +public class QuantityEncoding { + + @Nullable + public static Row encode(@Nullable final Quantity quantity) { + if (quantity == null) { + return null; + } + return RowFactory.create(quantity.getId(), quantity.getValue(), + quantity.getComparator().toCode(), quantity.getUnit(), quantity.getSystem(), + quantity.getCode(), null /* _fid */); + } + + @Nonnull + public static Quantity decode(@Nonnull final Row row) { + final Quantity quantity = new Quantity(); + quantity.setId(row.getString(0)); + quantity.setValue(row.getDecimal(1)); + quantity.setComparator(QuantityComparator.fromCode(row.getString(2))); + quantity.setUnit(row.getString(3)); + quantity.setSystem(row.getString(4)); + quantity.setCode(row.getString(5)); + return quantity; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 28d9eb33dd..e9c7fc78ff 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -7,9 +7,19 @@ package au.csiro.pathling.fhirpath.literal; import static au.csiro.pathling.utilities.Preconditions.check; +import static org.apache.spark.sql.functions.lit; +import static org.apache.spark.sql.functions.struct; +import au.csiro.pathling.errors.InvalidUserInputError; +import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.QuantityPath; +import au.csiro.pathling.terminology.ucum.Ucum; +import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -17,7 +27,9 @@ import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; import org.hl7.fhir.r4.model.Type; /** @@ -26,9 +38,29 @@ * @author John Grimes */ @Getter -public class QuantityLiteralPath extends LiteralPath { +public class QuantityLiteralPath extends LiteralPath implements Comparable { - private static final Pattern PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); + private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); + private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); + + private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("year", "a") + .put("years", "a") + .put("month", "mo") + .put("months", "mo") + .put("week", "wk") + .put("weeks", "wk") + .put("day", "d") + .put("days", "d") + .put("hour", "h") + .put("hours", "h") + .put("minute", "min") + .put("minutes", "min") + .put("second", "s") + .put("seconds", "s") + .put("millisecond", "ms") + .put("milliseconds", "ms") + .build(); @SuppressWarnings("WeakerAccess") protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @@ -40,39 +72,88 @@ protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull fina /** * Returns a new instance, parsed from a FHIRPath literal. * - * @param fhirPath The FHIRPath representation of the literal - * @param context An input context that can be used to build a {@link Dataset} to represent the + * @param fhirPath the FHIRPath representation of the literal + * @param context an input context that can be used to build a {@link Dataset} to represent the * literal + * @param ucumService a UCUM service for validating the unit within the literal * @return A new instance of {@link LiteralPath} * @throws IllegalArgumentException if the literal is malformed */ @Nonnull public static QuantityLiteralPath fromString(@Nonnull final String fhirPath, - @Nonnull final FhirPath context) throws IllegalArgumentException { - final Matcher matcher = PATTERN.matcher(fhirPath); - if (!matcher.matches()) { - throw new IllegalArgumentException("Quantity literal has invalid format: " + fhirPath); + @Nonnull final FhirPath context, final UcumService ucumService) + throws IllegalArgumentException { + final Matcher ucumMatcher = UCUM_PATTERN.matcher(fhirPath); + if (ucumMatcher.matches()) { + return fromUcumString(ucumMatcher, context, ucumService); + } else { + final Matcher calendarDurationMatcher = CALENDAR_DURATION_PATTERN.matcher(fhirPath); + if (calendarDurationMatcher.matches()) { + return fromCalendarDurationString(calendarDurationMatcher, context, ucumService); + } else { + throw new IllegalArgumentException("Quantity literal has invalid format: " + fhirPath); + } } + } + + @Nonnull + private static QuantityLiteralPath fromUcumString(@Nonnull final Matcher matcher, + @Nonnull final FhirPath context, @Nonnull final UcumService ucumService) { + final String fullPath = matcher.group(0); + final String value = matcher.group(1); final String rawUnit = matcher.group(2); final String unit = StringLiteralPath.fromString(rawUnit, context).getLiteralValue() .getValueAsString(); - final Quantity quantity = new Quantity(); - quantity.setUnit(unit); + final String validationResult = ucumService.validate(unit); + if (validationResult != null) { + throw new InvalidUserInputError( + "Invalid UCUM unit provided within Quantity literal (" + fullPath + "): " + + validationResult); + } + + final BigDecimal decimalValue = getQuantityValue(value, context); + final String display = ucumService.getCommonDisplay(unit); + + return buildLiteralPath(decimalValue, unit, display, context); + } + + @Nonnull + private static QuantityLiteralPath fromCalendarDurationString(@Nonnull final Matcher matcher, + @Nonnull final FhirPath context, final UcumService ucumService) { + final String value = matcher.group(1); + final String keyword = matcher.group(2); + + if (!CALENDAR_DURATION_TO_UCUM.containsKey(keyword)) { + throw new IllegalArgumentException("Invalid calendar duration keyword: " + keyword); + } + + final String unit = CALENDAR_DURATION_TO_UCUM.get(keyword); + final BigDecimal decimalValue = getQuantityValue(value, context); + final String display = ucumService.getCommonDisplay(unit); + + return buildLiteralPath(decimalValue, unit, display, context); + } + + private static BigDecimal getQuantityValue(final String value, final @Nonnull FhirPath context) { + final BigDecimal decimalValue; try { - final long value = IntegerLiteralPath.fromString(fhirPath, context) - .getLiteralValue().getValue(); - quantity.setValue(value); + decimalValue = DecimalLiteralPath.fromString(value, context).getLiteralValue().getValue(); } catch (final NumberFormatException e) { - try { - final BigDecimal value = DecimalLiteralPath.fromString(fhirPath, context) - .getLiteralValue().getValue(); - quantity.setValue(value); - } catch (final NumberFormatException ex) { - throw new IllegalArgumentException("Quantity literal has invalid format: " + fhirPath); - } + throw new IllegalArgumentException("Quantity literal has invalid value: " + value); } + return decimalValue; + } + + @Nonnull + private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, + final String unit, final String display, final @Nonnull FhirPath context) { + final Quantity quantity = new Quantity(); + quantity.setValue(decimalValue); + quantity.setSystem(Ucum.SYSTEM_URI); + quantity.setCode(unit); + quantity.setUnit(display); return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); } @@ -94,4 +175,30 @@ public Quantity getJavaValue() { return getLiteralValue(); } + @Nonnull + @Override + public Column buildValueColumn() { + final Quantity value = getJavaValue(); + final Optional comparator = Optional.ofNullable(value.getComparator()); + return struct( + lit(value.getId()).as("id"), + lit(value.getValue()).as("value"), + lit(comparator.map(QuantityComparator::toCode).orElse(null)).as("comparator"), + lit(value.getUnit()).as("unit"), + lit(value.getSystem()).as("system"), + lit(value.getCode()).as("code"), + lit(null).as("_fid")); + } + + @Nonnull + @Override + public Function getComparison(@Nonnull final ComparisonOperation operation) { + return QuantityPath.buildComparison(this, operation); + } + + @Override + public boolean isComparableTo(@Nonnull final Class type) { + return QuantityPath.COMPARABLE_TYPES.contains(type); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java index 8e33dc1cf4..b7ed1f140d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java @@ -10,12 +10,31 @@ import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.FhirPath; -import au.csiro.pathling.fhirpath.literal.*; +import au.csiro.pathling.fhirpath.literal.BooleanLiteralPath; +import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; +import au.csiro.pathling.fhirpath.literal.DecimalLiteralPath; +import au.csiro.pathling.fhirpath.literal.IntegerLiteralPath; +import au.csiro.pathling.fhirpath.literal.NullLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.fhirpath.literal.StringLiteralPath; +import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; import au.csiro.pathling.fhirpath.parser.generated.FhirPathBaseVisitor; -import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.*; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.BooleanLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.CodingLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.DateLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.DateTimeLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.NullLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.NumberLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.QuantityLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.StringLiteralContext; +import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.TimeLiteralContext; +import au.csiro.pathling.terminology.ucum.Ucum; import java.text.ParseException; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.fhir.ucum.UcumException; /** * This class deals with terms that are literal expressions. @@ -126,7 +145,18 @@ public FhirPath visitNullLiteral(@Nullable final NullLiteralContext ctx) { @Override @Nonnull public FhirPath visitQuantityLiteral(@Nullable final QuantityLiteralContext ctx) { - throw new InvalidUserInputError("Quantity literals are not supported"); + checkNotNull(ctx); + @Nullable final String number = ctx.quantity().NUMBER().getText(); + @Nullable final String unit = ctx.quantity().unit().getText(); + checkNotNull(number); + checkNotNull(unit); + final String fhirPath = String.format("%s %s", number, unit); + try { + return QuantityLiteralPath.fromString(fhirPath, + context.getThisContext().orElse(context.getInputContext()), Ucum.ucumEssenceService()); + } catch (UcumException e) { + throw new RuntimeException(e); + } } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index b286815c35..d40ec126b1 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -9,8 +9,9 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.Configuration.Storage.Aws; import au.csiro.pathling.async.SparkListener; -import au.csiro.pathling.sql.CodingToLiteral; import au.csiro.pathling.sql.PathlingStrategy; +import au.csiro.pathling.terminology.CodingToLiteral; +import au.csiro.pathling.terminology.ucum.QuantityEquality; import java.util.Arrays; import java.util.Objects; import java.util.Optional; @@ -50,7 +51,8 @@ public class Spark { @Nonnull public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, - @Nonnull final Optional sparkListener) { + @Nonnull final Optional sparkListener, + @Nonnull final QuantityEquality quantityEquality) { log.debug("Creating Spark session"); resolveSparkConfiguration(environment); @@ -63,6 +65,7 @@ public static SparkSession build(@Nonnull final Configuration configuration, PathlingStrategy.setup(spark); spark.udf() .register(CodingToLiteral.FUNCTION_NAME, new CodingToLiteral(), DataTypes.StringType); + spark.udf().register(QuantityEquality.FUNCTION_NAME, quantityEquality, DataTypes.BooleanType); // Configure AWS driver and credentials. configureAwsDriver(configuration, spark); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/CodingToLiteral.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java similarity index 96% rename from fhir-server/src/main/java/au/csiro/pathling/sql/CodingToLiteral.java rename to fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java index 37cf94469c..4248380d48 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/CodingToLiteral.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql; +package au.csiro.pathling.terminology; import au.csiro.pathling.fhirpath.encoding.CodingEncoding; import au.csiro.pathling.fhirpath.literal.CodingLiteral; diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java new file mode 100644 index 0000000000..08fd22be19 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.terminology.ucum; + +import au.csiro.pathling.fhirpath.element.DecimalPath; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.api.java.UDF2; +import org.fhir.ucum.Decimal; +import org.fhir.ucum.Pair; +import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class QuantityEquality implements UDF2 { + + @Nonnull + private final UcumService ucumService; + + private static final long serialVersionUID = -5960116379100195530L; + + /** + * The name of this function when used within SQL. + */ + public static final String FUNCTION_NAME = "quantity_equals"; + + public QuantityEquality(@Nonnull final UcumService ucumService) { + this.ucumService = ucumService; + } + + @Override + public Boolean call(final Row left, final Row right) throws Exception { + final Quantity leftQuantity = QuantityEncoding.decode(left); + final Quantity rightQuantity = QuantityEncoding.decode(right); + + final String leftSystem = leftQuantity.getSystem(); + final String rightSystem = rightQuantity.getSystem(); + final String leftCode = leftQuantity.getCode(); + final String rightCode = rightQuantity.getCode(); + if (!leftQuantity.hasValue() || !rightQuantity.hasValue() + || leftSystem == null || rightSystem == null + || leftCode == null || rightCode == null) { + return null; + } + + if (leftSystem.equals(rightSystem) && leftCode.equals(rightCode)) { + return leftQuantity.getValue().equals(rightQuantity.getValue()); + } + + final int maxPrecision = DecimalPath.getDecimalType().precision(); + final Decimal leftValue = new Decimal(leftQuantity.getValue().toPlainString(), maxPrecision); + final Decimal rightValue = new Decimal(rightQuantity.getValue().toPlainString(), maxPrecision); + if (leftSystem.equals(Ucum.SYSTEM_URI) && rightSystem.equals(Ucum.SYSTEM_URI) + && Objects.equals(leftQuantity.getComparator(), rightQuantity.getComparator()) + && ucumService.isComparable(leftCode, rightCode)) { + final Pair leftCanonical = ucumService.getCanonicalForm(new Pair(leftValue, leftCode)); + final Pair rightCanonical = ucumService.getCanonicalForm(new Pair(rightValue, rightCode)); + if (!leftCanonical.getCode().equals(rightCanonical.getCode())) { + log.warn("Encountered comparable canonical UCUM forms with different codes: {} and {}", + leftQuantity, rightQuantity); + return null; + } else { + return leftCanonical.getValue().equals(rightCanonical.getValue()); + } + } else { + return null; + } + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java new file mode 100644 index 0000000000..996e653b12 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.terminology.ucum; + +import java.io.InputStream; +import javax.annotation.Nonnull; +import org.fhir.ucum.UcumEssenceService; +import org.fhir.ucum.UcumException; +import org.fhir.ucum.UcumService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +/** + * Makes UCUM services available to the rest of the application. + * + * @author John Grimes + */ +@Component +@Profile({"core", "fhir"}) +public class Ucum { + + public static final String SYSTEM_URI = "http://unitsofmeasure.org/"; + + @Bean + @Nonnull + public static UcumService ucumEssenceService() throws UcumException { + final InputStream essenceStream = Ucum.class.getClassLoader() + .getResourceAsStream("tx/ucum-essence.xml"); + return new UcumEssenceService(essenceStream); + } + +} diff --git a/fhir-server/src/main/resources/tx/ucum-essence.xml b/fhir-server/src/main/resources/tx/ucum-essence.xml new file mode 100644 index 0000000000..d012996905 --- /dev/null +++ b/fhir-server/src/main/resources/tx/ucum-essence.xml @@ -0,0 +1,2110 @@ + + + + yotta + Y + 1 × 10 + 24 + + + + zetta + Z + 1 × 10 + 21 + + + + exa + E + 1 × 10 + 18 + + + + peta + P + 1 × 10 + 15 + + + + tera + T + 1 × 10 + 12 + + + + giga + G + 1 × 10 + 9 + + + + mega + M + 1 × 10 + 6 + + + + kilo + k + 1 × 10 + 3 + + + + hecto + h + 1 × 10 + 2 + + + + deka + da + 1 × 10 + 1 + + + + deci + d + 1 × 10 + -1 + + + + centi + c + 1 × 10 + -2 + + + + milli + m + 1 × 10 + -3 + + + + micro + μ + 1 × 10 + -6 + + + + nano + n + 1 × 10 + -9 + + + + pico + p + 1 × 10 + -12 + + + + femto + f + 1 × 10 + -15 + + + + atto + a + 1 × 10 + -18 + + + + zepto + z + 1 × 10 + -21 + + + + yocto + y + 1 × 10 + -24 + + + + meter + m + length + + + second + s + time + + + gram + g + mass + + + radian + rad + plane angle + + + kelvin + K + temperature + + + coulomb + C + electric charge + + + candela + cd + luminous intensity + + + the number ten for arbitrary powers + 10 + number + 10 + + + the number ten for arbitrary powers + 10 + number + 10 + + + the number pi + π + number + π + + + + percent + % + fraction + 1 + + + parts per thousand + ppth + fraction + 1 + + + parts per million + ppm + fraction + 1 + + + parts per billion + ppb + fraction + 1 + + + parts per trillion + pptr + fraction + 1 + + + mole + mol + amount of substance + 6.0221367 + + + steradian + sr + solid angle + 1 + + + hertz + Hz + frequency + 1 + + + newton + N + force + 1 + + + pascal + Pa + pressure + 1 + + + joule + J + energy + 1 + + + watt + W + power + 1 + + + ampère + A + electric current + 1 + + + volt + V + electric potential + 1 + + + farad + F + electric capacitance + 1 + + + ohm + Ω + electric resistance + 1 + + + siemens + S + electric conductance + 1 + + + weber + Wb + magnetic flux + 1 + + + degree Celsius + °C + temperature + + + + + + tesla + T + magnetic flux density + 1 + + + henry + H + inductance + 1 + + + lumen + lm + luminous flux + 1 + + + lux + lx + illuminance + 1 + + + becquerel + Bq + radioactivity + 1 + + + gray + Gy + energy dose + 1 + + + sievert + Sv + dose equivalent + 1 + + + gon + grade + □g + + plane angle + 0.9 + + + degree + ° + plane angle + 2 + + + minute + ' + plane angle + 1 + + + second + '' + plane angle + 1 + + + liter + l + volume + 1 + + + liter + L + volume + 1 + + + are + a + area + 100 + + + minute + min + time + 60 + + + hour + h + time + 60 + + + day + d + time + 24 + + + tropical year + a + t + + time + 365.24219 + + + mean Julian year + a + j + + time + 365.25 + + + mean Gregorian year + a + g + + time + 365.2425 + + + year + a + time + 1 + + + week + wk + time + 7 + + + synodal month + mo + s + + time + 29.53059 + + + mean Julian month + mo + j + + time + 1 + + + mean Gregorian month + mo + g + + time + 1 + + + month + mo + time + 1 + + + tonne + t + mass + 1 × 10 + 3 + + + + bar + bar + pressure + 1 × 10 + 5 + + + + unified atomic mass unit + u + mass + 1.6605402 × 10 + -24 + + + + electronvolt + eV + energy + 1 + + + astronomic unit + AU + length + 149597.870691 + + + parsec + pc + length + 3.085678 × 10 + 16 + + + + velocity of light + + c + + velocity + 299792458 + + + Planck constant + + h + + action + 6.6260755 × 10 + -34 + + + + Boltzmann constant + + k + + (unclassified) + 1.380658 × 10 + -23 + + + + permittivity of vacuum + + ε + 0 + + + + electric permittivity + 8.854187817 × 10 + -12 + + + + permeability of vacuum + + μ + 0 + + + + magnetic permeability + 1 + + + elementary charge + + e + + electric charge + 1.60217733 × 10 + -19 + + + + electron mass + + m + + e + + + + mass + 9.1093897 × 10 + -28 + + + + proton mass + + m + + p + + + + mass + 1.6726231 × 10 + -24 + + + + Newtonian constant of gravitation + + G + + (unclassified) + 6.67259 × 10 + -11 + + + + standard acceleration of free fall + + g + n + + + acceleration + 9.80665 + + + standard atmosphere + atm + pressure + 101325 + + + light-year + l.y. + length + 1 + + + gram-force + gf + force + 1 + + + pound force + lbf + force + 1 + + + Kayser + K + lineic number + 1 + + + Gal + Gal + acceleration + 1 + + + dyne + dyn + force + 1 + + + erg + erg + energy + 1 + + + Poise + P + dynamic viscosity + 1 + + + Biot + Bi + electric current + 10 + + + Stokes + St + kinematic viscosity + 1 + + + Maxwell + Mx + flux of magnetic induction + 1 × 10 + -8 + + + + Gauss + Gs + magnetic flux density + 1 × 10 + -4 + + + + Oersted + Oe + magnetic field intensity + 250 + + + Gilbert + Gb + magnetic tension + 1 + + + stilb + sb + lum. intensity density + 1 + + + Lambert + L + brightness + 1 + + + phot + ph + illuminance + 1 × 10 + -4 + + + + Curie + Ci + radioactivity + 3.7 × 10 + 10 + + + + Roentgen + R + ion dose + 2.58 × 10 + -4 + + + + radiation absorbed dose + RAD + energy dose + 100 + + + radiation equivalent man + REM + dose equivalent + 1 + + + inch + in + length + 2.54 + + + foot + ft + length + 12 + + + yard + yd + length + 3 + + + mile + mi + length + 5280 + + + fathom + fth + depth of water + 6 + + + nautical mile + n.mi + length + 1852 + + + knot + knot + velocity + 1 + + + square inch + area + 1 + + + square foot + area + 1 + + + square yard + area + 1 + + + cubic inch + volume + 1 + + + cubic foot + volume + 1 + + + cubic yard + cu.yd + volume + 1 + + + board foot + volume + 144 + + + cord + volume + 128 + + + mil + mil + length + 1 × 10 + -3 + + + + circular mil + circ.mil + area + 1 + + + hand + hd + height of horses + 4 + + + foot + ft + us + + length + 1200 + + + yard + length + 3 + + + inch + length + 1 + + + rod + length + 16.5 + + + Gunter's chain + Surveyor's chain + length + 4 + + + link for Gunter's chain + length + 1 + + + Ramden's chain + Engineer's chain + length + 100 + + + link for Ramden's chain + length + 1 + + + fathom + length + 6 + + + furlong + length + 40 + + + mile + length + 8 + + + acre + area + 160 + + + square rod + area + 1 + + + square mile + area + 1 + + + section + area + 1 + + + township + area + 36 + + + mil + length + 1 × 10 + -3 + + + + inch + length + 2.539998 + + + foot + length + 12 + + + rod + length + 16.5 + + + Gunter's chain + length + 4 + + + link for Gunter's chain + length + 1 + + + fathom + length + 6 + + + pace + length + 2.5 + + + yard + length + 3 + + + mile + length + 5280 + + + nautical mile + length + 6080 + + + knot + velocity + 1 + + + acre + area + 4840 + + + Queen Anne's wine gallon + fluid volume + 231 + + + barrel + fluid volume + 42 + + + quart + fluid volume + 1 + + + pint + fluid volume + 1 + + + gill + fluid volume + 1 + + + fluid ounce + oz fl + fluid volume + 1 + + + fluid dram + fluid volume + 1 + + + minim + fluid volume + 1 + + + cord + fluid volume + 128 + + + bushel + dry volume + 2150.42 + + + historical winchester gallon + dry volume + 1 + + + peck + dry volume + 1 + + + dry quart + dry volume + 1 + + + dry pint + dry volume + 1 + + + tablespoon + volume + 1 + + + teaspoon + volume + 1 + + + cup + volume + 16 + + + metric fluid ounce + oz fl + fluid volume + 30 + + + metric cup + volume + 240 + + + metric teaspoon + volume + 5 + + + metric tablespoon + volume + 15 + + + gallon + volume + 4.54609 + + + peck + volume + 2 + + + bushel + volume + 4 + + + quart + volume + 1 + + + pint + volume + 1 + + + gill + volume + 1 + + + fluid ounce + volume + 1 + + + fluid dram + volume + 1 + + + minim + volume + 1 + + + grain + mass + 64.79891 + + + pound + lb + mass + 7000 + + + ounce + oz + mass + 1 + + + dram + mass + 1 + + + short hundredweight + U.S. hundredweight + mass + 100 + + + long hunderdweight + British hundredweight + mass + 112 + + + short ton + U.S. ton + mass + 20 + + + long ton + British ton + mass + 20 + + + stone + British stone + mass + 14 + + + pennyweight + mass + 24 + + + ounce + mass + 20 + + + pound + mass + 12 + + + scruple + mass + 20 + + + dram + drachm + mass + 3 + + + ounce + mass + 8 + + + pound + mass + 12 + + + metric ounce + mass + 28 + + + line + length + 1 + + + point + length + 1 + + + pica + length + 12 + + + Printer's point + length + 0.013837 + + + Printer's pica + length + 12 + + + pied + French foot + length + 32.48 + + + pouce + French inch + length + 1 + + + ligne + French line + length + 1 + + + didot + Didot's point + length + 1 + + + cicero + Didot's pica + length + 12 + + + degree Fahrenheit + °F + temperature + + + + + + degree Rankine + °R + temperature + 5 + + + degree Réaumur + °Ré + temperature + + + + + + calorie at 15 °C + cal + 15°C + + energy + 4.18580 + + + calorie at 20 °C + cal + 20°C + + energy + 4.18190 + + + mean calorie + cal + m + + energy + 4.19002 + + + international table calorie + cal + IT + + energy + 4.1868 + + + thermochemical calorie + cal + th + + energy + 4.184 + + + calorie + cal + energy + 1 + + + nutrition label Calories + Cal + energy + 1 + + + British thermal unit at 39 °F + Btu + 39°F + + energy + 1.05967 + + + British thermal unit at 59 °F + Btu + 59°F + + energy + 1.05480 + + + British thermal unit at 60 °F + Btu + 60°F + + energy + 1.05468 + + + mean British thermal unit + Btu + m + + energy + 1.05587 + + + international table British thermal unit + Btu + IT + + energy + 1.05505585262 + + + thermochemical British thermal unit + Btu + th + + energy + 1.054350 + + + British thermal unit + btu + energy + 1 + + + horsepower + power + 550 + + + tex + tex + linear mass density (of textile thread) + 1 + + + Denier + den + linear mass density (of textile thread) + 1 + + + meter of water column + m H + + 2 + + O + + pressure + 9.80665 + + + meter of mercury column + m Hg + pressure + 133.3220 + + + inch of water column + in H + + 2 + + O + + pressure + 1 + + + inch of mercury column + in Hg + pressure + 1 + + + peripheral vascular resistance unit + P.R.U. + fluid resistance + 1 + + + Wood unit + Wood U. + fluid resistance + 1 + + + diopter + dpt + refraction of a lens + 1 + + + prism diopter + PD + refraction of a prism + + + + + + percent of slope + % + slope + + + + + + mesh + lineic number + 1 + + + Charrière + french + Ch + gauge of catheters + 1 + + + drop + drp + volume + 1 + + + Hounsfield unit + HF + x-ray attenuation + 1 + + + metabolic equivalent + MET + metabolic cost of physical activity + 3.5 + + + homeopathic potency of decimal series (retired) + X + homeopathic potency (retired) + + + + + + homeopathic potency of centesimal series (retired) + C + homeopathic potency (retired) + + + + + + homeopathic potency of millesimal series (retired) + M + homeopathic potency (retired) + + + + + + homeopathic potency of quintamillesimal series (retired) + Q + homeopathic potency (retired) + + + + + + homeopathic potency of decimal hahnemannian series + X + homeopathic potency (Hahnemann) + 1 + + + homeopathic potency of centesimal hahnemannian series + C + homeopathic potency (Hahnemann) + 1 + + + homeopathic potency of millesimal hahnemannian series + M + homeopathic potency (Hahnemann) + 1 + + + homeopathic potency of quintamillesimal hahnemannian series + Q + homeopathic potency (Hahnemann) + 1 + + + homeopathic potency of decimal korsakovian series + X + homeopathic potency (Korsakov) + 1 + + + homeopathic potency of centesimal korsakovian series + C + homeopathic potency (Korsakov) + 1 + + + homeopathic potency of millesimal korsakovian series + M + homeopathic potency (Korsakov) + 1 + + + homeopathic potency of quintamillesimal korsakovian series + Q + homeopathic potency (Korsakov) + 1 + + + equivalents + eq + amount of substance + 1 + + + osmole + osm + amount of substance (dissolved particles) + 1 + + + pH + pH + acidity + + + + + + gram percent + g% + mass concentration + 1 + + + Svedberg unit + S + sedimentation coefficient + 1 + + + high power field + HPF + view area in microscope + 1 + + + low power field + LPF + view area in microscope + 100 + + + katal + kat + catalytic activity + 1 + + + Unit + U + catalytic activity + 1 + + + international unit + IU + arbitrary + 1 + + + international unit + i.U. + arbitrary + 1 + + + arbitary unit + arb. U + arbitrary + 1 + + + United States Pharmacopeia unit + U.S.P. + arbitrary + 1 + + + GPL unit + biologic activity of anticardiolipin IgG + 1 + + + MPL unit + biologic activity of anticardiolipin IgM + 1 + + + APL unit + biologic activity of anticardiolipin IgA + 1 + + + Bethesda unit + biologic activity of factor VIII inhibitor + 1 + + + anti factor Xa unit + biologic activity of factor Xa inhibitor (heparin) + 1 + + + Todd unit + biologic activity antistreptolysin O + 1 + + + Dye unit + biologic activity of amylase + 1 + + + Somogyi unit + biologic activity of amylase + 1 + + + Bodansky unit + biologic activity of phosphatase + 1 + + + King-Armstrong unit + biologic activity of phosphatase + 1 + + + Kunkel unit + arbitrary biologic activity + 1 + + + Mac Lagan unit + arbitrary biologic activity + 1 + + + tuberculin unit + biologic activity of tuberculin + 1 + + + 50% cell culture infectious dose + CCID + 50 + + biologic activity (infectivity) of an infectious agent preparation + 1 + + + 50% tissue culture infectious dose + TCID + 50 + + biologic activity (infectivity) of an infectious agent preparation + 1 + + + 50% embryo infectious dose + EID + 50 + + biologic activity (infectivity) of an infectious agent preparation + 1 + + + plaque forming units + PFU + amount of an infectious agent + 1 + + + focus forming units + FFU + amount of an infectious agent + 1 + + + colony forming units + CFU + amount of a proliferating organism + 1 + + + index of reactivity + IR + amount of an allergen callibrated through in-vivo testing using the Stallergenes® + method. + + 1 + + + bioequivalent allergen unit + BAU + amount of an allergen callibrated through in-vivo testing based on the ID50EAL method + of (intradermal dilution for 50mm sum of erythema diameters + + 1 + + + allergen unit + AU + procedure defined amount of an allergen using some reference standard + 1 + + + allergen unit for Ambrosia artemisiifolia + Amb a 1 U + procedure defined amount of the major allergen of ragweed. + 1 + + + protein nitrogen unit + PNU + procedure defined amount of a protein substance + 1 + + + Limit of flocculation + Lf + procedure defined amount of an antigen substance + 1 + + + D-antigen unit + + procedure defined amount of a poliomyelitis d-antigen substance + 1 + + + fibrinogen equivalent unit + + amount of fibrinogen broken down into the measured d-dimers + 1 + + + ELISA unit + + arbitrary ELISA unit + 1 + + + Ehrlich unit + + Ehrlich unit + 1 + + + neper + Np + level + + + + + + bel + B + level + + + + + + bel sound pressure + B(SPL) + pressure level + + + + + + bel volt + B(V) + electric potential level + + + + + + bel millivolt + B(mV) + electric potential level + + + + + + bel microvolt + B(μV) + electric potential level + + + + + + bel 10 nanovolt + B(10 nV) + electric potential level + + + + + + bel watt + B(W) + power level + + + + + + bel kilowatt + B(kW) + power level + + + + + + stere + st + volume + 1 + + + Ångström + Å + length + 0.1 + + + barn + b + action area + 100 + + + technical atmosphere + at + pressure + 1 + + + mho + mho + electric conductance + 1 + + + pound per sqare inch + psi + pressure + 1 + + + circle + circ + plane angle + 2 + + + spere + sph + solid angle + 4 + + + metric carat + ct + m + + mass + 0.2 + + + carat of gold alloys + ct + + Au + + + mass fraction + 1 + + + Smoot + + length + 67 + + + meter per square seconds per square root of hertz + + amplitude spectral density + + + + + + bit + bit + s + + amount of information + + + + + + bit + bit + amount of information + 1 + + + byte + B + amount of information + 8 + + + baud + Bd + signal transmission rate + 1 + + + kibi + Ki + 1024 + + + mebi + Mi + 1048576 + + + gibi + Gi + 1073741824 + + + tebi + Ti + 1099511627776 + + \ No newline at end of file diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 59f1471955..f9149267c4 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -13,12 +13,16 @@ import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; import au.csiro.pathling.terminology.TerminologyService; +import au.csiro.pathling.terminology.ucum.QuantityEquality; +import au.csiro.pathling.terminology.ucum.Ucum; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import java.util.Optional; import javax.annotation.Nonnull; import org.apache.spark.sql.SparkSession; +import org.fhir.ucum.UcumException; +import org.fhir.ucum.UcumService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Profile; @@ -37,8 +41,9 @@ class UnitTestDependencies { @Nonnull static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, - @Nonnull final Optional sparkListener) { - return Spark.build(configuration, environment, sparkListener); + @Nonnull final Optional sparkListener, + @Nonnull final QuantityEquality quantityEquality) { + return Spark.build(configuration, environment, sparkListener, quantityEquality); } @Bean @@ -83,4 +88,11 @@ static TerminologyServiceFactory terminologyClientFactory() { return new TestTerminologyServiceFactory(); } + @Bean + @ConditionalOnMissingBean + @Nonnull + static UcumService ucumService() throws UcumException { + return Ucum.ucumEssenceService(); + } + } From 2750f4f2094b47b069daece1fd7ea90e1854af15 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 1 Apr 2022 11:30:44 +1000 Subject: [PATCH 02/83] Add documentation for targeted functionality --- site/docs/fhirpath/data-types.md | 46 ++++++++++++++++++++++++-- site/docs/fhirpath/operators.md | 57 ++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index b8a02dd94b..f72f445867 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -18,6 +18,7 @@ literal expressions: - [Date](#date) - [DateTime](#datetime) - [Time](#time) +- [Quantity](#quantity) - [Coding](#coding) See also: [Literals](https://hl7.org/fhirpath/#literals) and @@ -138,6 +139,41 @@ Example: @2015-02-08T13:28:17-05:00 ``` +## Quantity + +The Quantity type represents quantities with a specified unit, where the value +component is defined as a Decimal, and the unit element is represented as a +String that is required to be either a valid +[Unified Code for Units of Measure (UCUM)](https://ucum.org/ucum.html) unit or +one of the calendar duration keywords, singular or plural. + +The Quantity literal is a number (integer or decimal), followed by a +(single-quoted) string representing a valid UCUM unit or calendar duration +keyword. If the value literal is an Integer, it will be implicitly converted to +a Decimal in the resulting Quantity value. + +The calendar duration keywords that are supported are: + +- `year` / `years` +- `month` / `months` +- `week` / `weeks` +- `day` / `days` +- `hour` / `hours` +- `minute` / `minutes` +- `second` / `seconds` +- `millisecond` / `milliseconds` + +Example: + +``` +4.5 'mg' +100 '[degF]' +6 months +30 days +``` + +See: [Quantity](https://hl7.org/fhirpath/#quantity) + ## Coding A [Coding](https://hl7.org/fhir/R4/datatypes.html#Coding) is a representation of @@ -146,7 +182,8 @@ a defined concept using a symbol from a defined [Using Codes in resources](https://hl7.org/fhir/R4/terminologies.html) for more details. -The Coding literal comprises a minimum of `system` and `code`, as well as optional `version`, `display`, `userSelected` components: +The Coding literal comprises a minimum of `system` and `code`, as well as +optional `version`, `display`, `userSelected` components: ``` |[|][|[|]]] @@ -172,7 +209,12 @@ http://snomed.info/sct|'397956004 |Prosthetic arthroplasty of the hip|: 36370400 ## Materializable types -There is a subset of all possible FHIR types that can be "materialized", i.e. used as the result of a grouping expression in the Aggregate operation, or the definition of a column within the Extract operation. These types are: +There is a subset of all possible FHIR types that can be "materialized", i.e. +used as the result of a grouping expression in +the [aggregate](../operations/aggregate.html) +operation, or the definition of a column within +the [extract](../operations/extract.html) +operation. These types are: - [Boolean](#boolean) - [String](#string) diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index a7258b6bc4..a0a55e7f77 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -17,6 +17,7 @@ Pathling: - [Comparison](#comparison) (`<=`, `<`, `>` and `>=`) - [Equality](#equality) (`=` and `!=`) - [Math](#math) (`+`, `-`, `*`, `/` and `mod`) +- [Date/time arithmetic](#datetime-arithmetic) (`+` and `-`) - [Boolean logic](#boolean-logic) (`and`, `or`, `xor` and `implies`) - [Membership](#membership) (`in` and `contains`) - [combine](#combine) @@ -35,15 +36,16 @@ The following comparison operators are supported: Both operands must be singular, the table below shows the valid types and their combinations. -| | Boolean | String | Integer | Decimal | Date | DateTime | Time | -|----------|---------|--------|---------|---------|-------|----------|-------| -| Boolean | true | false | false | false | false | false | false | -| String | false | true | false | false | false | false | false | -| Integer | false | false | true | true | false | false | false | -| Decimal | false | false | true | true | false | false | false | -| Date | false | false | false | false | true | true | false | -| DateTime | false | false | false | false | true | true | false | -| Time | false | false | false | false | false | false | true | +| | Boolean | String | Integer | Decimal | Date | DateTime | Time | Quantity | +|----------|---------|--------|---------|---------|-------|----------|-------|------------------| +| Boolean | true | false | false | false | false | false | false | false | +| String | false | true | false | false | false | false | false | false | +| Integer | false | false | true | true | false | false | false | false | +| Decimal | false | false | true | true | false | false | false | false | +| Date | false | false | false | false | true | true | false | true* | +| DateTime | false | false | false | false | true | true | false | true* | +| Time | false | false | false | false | false | false | true | false | +| Quantity | false | false | false | false | false | false | false | true* | If one or both of the operands is an empty collection, the operator will return an empty collection. @@ -53,6 +55,11 @@ individual characters. All comparison operators return a [Boolean](./data-types.html#boolean) value. +* Not all Quantity, Date and DateTime values are comparable, it +depends upon the comparability of the units within the Quantity values. See the +[FHIRPath specification](https://hl7.org/fhirpath/#comparison) for details on +how Quantity values are compared. + See also: [Comparison](https://hl7.org/fhirpath/#comparison) ## Equality @@ -60,14 +67,19 @@ See also: [Comparison](https://hl7.org/fhirpath/#comparison) The `=` operator returns `true` if the left operand is equal to the right operand, and a `false` otherwise. The `!=` is the inverse of the `=` operator. -Both operands must be singular. The valid types and their combinations is the -same as for the [Comparison operators](#comparison). In addition to this, -[Coding](./data-types.html#coding) types can -be compared using the equality operators. +Both operands must be singular. The valid types and their combinations is the +same as for the [Comparison operators](#comparison). In addition to this, +[Coding](./data-types.html#coding) types can be compared using the equality +operators. If one or both of the operands is an empty collection, the operator will return an empty collection. +Not all Quantity, Date and DateTime values can be compared for equality, it +depends upon the comparability of the units within the Quantity values. See the +[FHIRPath specification](https://hl7.org/fhirpath/#quantity-equality) for +details on how equality works with Quantity values. + See also: [Equality](https://hl7.org/fhirpath/#equality) ## Math @@ -94,6 +106,25 @@ an empty collection. See also: [Math](https://hl7.org/fhirpath/#math) +## Date/time arithmetic + +The following operators are supported for date arithmetic: + +- `+` - Add a duration to a [Date](./data-types.html#date) + or [DateTime](./data-types.html#datetime) +- `-` - Subtract a duration from a [Date](./data-types.html#date) + or [DateTime](./data-types.html#datetime) + +The duration operand is a [Quantity](./data-types.html#quantity) literal, and +must be a [calendar duration](https://hl7.org/fhirpath/#time-valued-quantities) +as defined in the FHIRPath specification. The use of UCUM units is not supported +with these operators. + +The Date or DateTime operand must be singular. If it is an empty collection, the +operator will return an empty collection. + +See also: [Date/Time Arithmetic](https://hl7.org/fhirpath/#datetime-arithmetic) + ## Boolean logic The following Boolean operations are supported: From c89cba5a7e4b08639e83c9aba19ae59329f0f282 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 4 Apr 2022 10:15:05 +1000 Subject: [PATCH 03/83] Add EqualityOperatorQuantityTest --- .../fhirpath/element/QuantityPath.java | 4 +- .../literal/CalendarDurationLiteralPath.java | 46 ++ .../fhirpath/literal/LiteralPath.java | 4 +- ...Path.java => UcumQuantityLiteralPath.java} | 15 +- .../fhirpath/parser/LiteralTermVisitor.java | 6 +- .../EqualityOperatorQuantityTest.java | 438 ++++++++++++++++++ .../pathling/test/helpers/SparkHelpers.java | 28 +- .../pathling/test/helpers/TestHelpers.java | 1 + 8 files changed, 527 insertions(+), 15 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java rename fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/{QuantityLiteralPath.java => UcumQuantityLiteralPath.java} (91%) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index f899c3bdbe..76db401c51 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -10,7 +10,7 @@ import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; import au.csiro.pathling.terminology.ucum.QuantityEquality; import com.google.common.collect.ImmutableSet; import java.util.Optional; @@ -30,7 +30,7 @@ public class QuantityPath extends ElementPath implements Comparable { public static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet - .of(QuantityPath.class, QuantityLiteralPath.class, NullLiteralPath.class); + .of(QuantityPath.class, UcumQuantityLiteralPath.class, NullLiteralPath.class); protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Optional eidColumn, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java new file mode 100644 index 0000000000..1f74ac64f9 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.literal; + +import au.csiro.pathling.fhirpath.FhirPath; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Type; + +public class CalendarDurationLiteralPath extends LiteralPath implements Comparable { + + protected CalendarDurationLiteralPath( + @Nonnull final Dataset dataset, + @Nonnull final Column idColumn, + @Nonnull final Type literalValue) { + super(dataset, idColumn, literalValue); + } + + public static CalendarDurationLiteralPath fromString(final String s, final FhirPath left) { + return null; + } + + @Nonnull + @Override + public String getExpression() { + return null; + } + + @Nullable + @Override + public Object getJavaValue() { + return null; + } + + @Override + public int compareTo(@Nonnull final Object o) { + return 0; + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java index 850d4d1124..8f1ae98471 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java @@ -55,7 +55,7 @@ public abstract class LiteralPath implements FhirPath { .put(FHIRDefinedType.INSTANT, DateTimeLiteralPath.class) .put(FHIRDefinedType.TIME, TimeLiteralPath.class) .put(FHIRDefinedType.CODING, CodingLiteralPath.class) - .put(FHIRDefinedType.QUANTITY, QuantityLiteralPath.class) + .put(FHIRDefinedType.QUANTITY, UcumQuantityLiteralPath.class) .build(); private static final Map, FHIRDefinedType> FHIRPATH_TYPE_TO_FHIR_TYPE = @@ -68,7 +68,7 @@ public abstract class LiteralPath implements FhirPath { .put(DateTimeLiteralPath.class, FHIRDefinedType.DATETIME) .put(TimeLiteralPath.class, FHIRDefinedType.TIME) .put(CodingLiteralPath.class, FHIRDefinedType.CODING) - .put(QuantityLiteralPath.class, FHIRDefinedType.QUANTITY) + .put(UcumQuantityLiteralPath.class, FHIRDefinedType.QUANTITY) .build(); @Getter diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java similarity index 91% rename from fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java rename to fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java index e9c7fc78ff..61023a255d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java @@ -38,7 +38,7 @@ * @author John Grimes */ @Getter -public class QuantityLiteralPath extends LiteralPath implements Comparable { +public class UcumQuantityLiteralPath extends LiteralPath implements Comparable { private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); @@ -63,7 +63,8 @@ public class QuantityLiteralPath extends LiteralPath implements Comparable { .build(); @SuppressWarnings("WeakerAccess") - protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + protected UcumQuantityLiteralPath(@Nonnull final Dataset dataset, + @Nonnull final Column idColumn, @Nonnull final Type literalValue) { super(dataset, idColumn, literalValue); check(literalValue instanceof Quantity); @@ -80,7 +81,7 @@ protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull fina * @throws IllegalArgumentException if the literal is malformed */ @Nonnull - public static QuantityLiteralPath fromString(@Nonnull final String fhirPath, + public static UcumQuantityLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context, final UcumService ucumService) throws IllegalArgumentException { final Matcher ucumMatcher = UCUM_PATTERN.matcher(fhirPath); @@ -98,7 +99,7 @@ public static QuantityLiteralPath fromString(@Nonnull final String fhirPath, } @Nonnull - private static QuantityLiteralPath fromUcumString(@Nonnull final Matcher matcher, + private static UcumQuantityLiteralPath fromUcumString(@Nonnull final Matcher matcher, @Nonnull final FhirPath context, @Nonnull final UcumService ucumService) { final String fullPath = matcher.group(0); final String value = matcher.group(1); @@ -120,7 +121,7 @@ private static QuantityLiteralPath fromUcumString(@Nonnull final Matcher matcher } @Nonnull - private static QuantityLiteralPath fromCalendarDurationString(@Nonnull final Matcher matcher, + private static UcumQuantityLiteralPath fromCalendarDurationString(@Nonnull final Matcher matcher, @Nonnull final FhirPath context, final UcumService ucumService) { final String value = matcher.group(1); final String keyword = matcher.group(2); @@ -147,7 +148,7 @@ private static BigDecimal getQuantityValue(final String value, final @Nonnull Fh } @Nonnull - private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, + private static UcumQuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, final String unit, final String display, final @Nonnull FhirPath context) { final Quantity quantity = new Quantity(); quantity.setValue(decimalValue); @@ -155,7 +156,7 @@ private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValu quantity.setCode(unit); quantity.setUnit(display); - return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); + return new UcumQuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); } @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java index b7ed1f140d..9372e4c868 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java @@ -17,9 +17,9 @@ import au.csiro.pathling.fhirpath.literal.DecimalLiteralPath; import au.csiro.pathling.fhirpath.literal.IntegerLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.literal.StringLiteralPath; import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; +import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.generated.FhirPathBaseVisitor; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.BooleanLiteralContext; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.CodingLiteralContext; @@ -152,9 +152,9 @@ public FhirPath visitQuantityLiteral(@Nullable final QuantityLiteralContext ctx) checkNotNull(unit); final String fhirPath = String.format("%s %s", number, unit); try { - return QuantityLiteralPath.fromString(fhirPath, + return UcumQuantityLiteralPath.fromString(fhirPath, context.getThisContext().orElse(context.getInputContext()), Ucum.ucumEssenceService()); - } catch (UcumException e) { + } catch (final UcumException e) { throw new RuntimeException(e); } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java new file mode 100644 index 0000000000..f409c65dee --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java @@ -0,0 +1,438 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static au.csiro.pathling.test.helpers.SparkHelpers.quantityStructType; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; +import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.TestHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.Collections; +import java.util.List; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.hl7.fhir.r4.model.Quantity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +@Tag("UnitTest") +public class EqualityOperatorQuantityTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + @Autowired + UcumService ucumService; + + static final String ID_ALIAS = "_abc123"; + + FhirPath left; + FhirPath right; + ParserContext parserContext; + UcumQuantityLiteralPath ucumQuantityLiteral1; + UcumQuantityLiteralPath ucumQuantityLiteral2; + UcumQuantityLiteralPath ucumQuantityLiteral3; + CalendarDurationLiteralPath calendarDurationLiteral1; + CalendarDurationLiteralPath calendarDurationLiteral2; + CalendarDurationLiteralPath calendarDurationLiteral3; + + @BeforeEach + void setUp() { + final Quantity quantity1 = new Quantity(); + quantity1.setValue(500); + quantity1.setUnit("mg"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("mg"); + + final Quantity quantity2 = new Quantity(); + quantity2.setValue(0.5); + quantity2.setUnit("g"); + quantity2.setSystem(TestHelpers.UCUM_URL); + quantity2.setCode("g"); + + final Quantity quantity3 = new Quantity(); + quantity3.setValue(1.8); + quantity3.setUnit("m"); + quantity3.setSystem(TestHelpers.UCUM_URL); + quantity3.setCode("m"); + + final Quantity quantity4 = new Quantity(); + quantity4.setValue(0.5); + quantity4.setUnit("g"); + quantity4.setSystem(TestHelpers.SNOMED_URL); + quantity4.setCode("258682000"); + + final Quantity quantity5 = new Quantity(); + quantity1.setValue(650); + quantity1.setUnit("mg"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("mg"); + + final Quantity quantity6 = new Quantity(); + quantity1.setValue(30); + quantity1.setUnit("d"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("d"); + + final Quantity quantity7 = new Quantity(); + quantity1.setValue(60); + quantity1.setUnit("s"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("s"); + + final Quantity quantity8 = new Quantity(); + quantity1.setValue(1000); + quantity1.setUnit("ms"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("ms"); + + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-2", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-3", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-4", rowFromQuantity(quantity5)) // 650 mg + .withRow("patient-5", null) + .withRow("patient-6", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .buildWithStructValue(); + left = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-2", rowFromQuantity(quantity2)) // 0.5 g + .withRow("patient-3", rowFromQuantity(quantity3)) // 1.8 m + .withRow("patient-4", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-5", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-6", null) + .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .buildWithStructValue(); + right = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(rightDataset) + .idAndValueColumns() + .build(); + + ucumQuantityLiteral1 = UcumQuantityLiteralPath.fromString("500 'mg'", left, ucumService); + ucumQuantityLiteral2 = UcumQuantityLiteralPath.fromString("0.5 'g'", left, ucumService); + ucumQuantityLiteral3 = UcumQuantityLiteralPath.fromString("1.8 'm'", left, ucumService); + calendarDurationLiteral1 = CalendarDurationLiteralPath.fromString("30 days", left); + calendarDurationLiteral2 = CalendarDurationLiteralPath.fromString("60 seconds", left); + calendarDurationLiteral3 = CalendarDurationLiteralPath.fromString("1000 milliseconds", left); + + parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + } + + @Test + void equals() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance("="); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // 500 mg = 500 mg + RowFactory.create("patient-2", true), // 500 mg = 0.5 g + RowFactory.create("patient-3", null), // 500 mg = 1.8 m + RowFactory.create("patient-4", false), // 650 mg = 500 mg + RowFactory.create("patient-5", null), // {} = 500 mg + RowFactory.create("patient-6", null), // 500 mg = {} + RowFactory.create("patient-7", true), // 30 d = 30 d + RowFactory.create("patient-8", true), // 60 s = 60 s + RowFactory.create("patient-9", true) // 1000 ms = 1000 ms + ); + } + + @Test + void notEquals() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance("!="); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // 500 mg != 500 mg + RowFactory.create("patient-2", false), // 500 mg != 0.5 g + RowFactory.create("patient-3", null), // 500 mg != 1.8 m + RowFactory.create("patient-4", true), // 650 mg != 500 mg + RowFactory.create("patient-5", null), // {} != 500 mg + RowFactory.create("patient-6", null), // 500 mg != {} + RowFactory.create("patient-7", false), // 30 d != 30 d + RowFactory.create("patient-8", false), // 60 s != 60 s + RowFactory.create("patient-9", false) // 1000 ms != 1000 ms + ); + } + + @Test + void ucumLiteralEquals() { + OperatorInput input = new OperatorInput(parserContext, left, ucumQuantityLiteral1); + final Operator equalityOperator = Operator.getInstance("="); + FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // 500 mg = 500 mg + RowFactory.create("patient-2", true), // 500 mg = 500 mg + RowFactory.create("patient-3", true), // 500 mg = 500 mg + RowFactory.create("patient-4", false), // 650 mg = 500 mg + RowFactory.create("patient-5", null), // {} = 500 mg + RowFactory.create("patient-6", true), // 500 mg = 500 mg + RowFactory.create("patient-7", null), // 30 d = 500 mg + RowFactory.create("patient-8", null), // 60 s = 500 mg + RowFactory.create("patient-9", null) // 1000 ms = 500 mg + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteral2); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // 500 mg = 0.5 mg + RowFactory.create("patient-2", true), // 500 mg = 0.5 mg + RowFactory.create("patient-3", true), // 500 mg = 0.5 mg + RowFactory.create("patient-4", false), // 650 mg = 0.5 mg + RowFactory.create("patient-5", null), // {} = 0.5 mg + RowFactory.create("patient-6", true), // 500 mg = 0.5 mg + RowFactory.create("patient-7", null), // 30 d = 0.5 mg + RowFactory.create("patient-8", null), // 60 s = 0.5 mg + RowFactory.create("patient-9", null) // 1000 ms = 0.5 mg + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteral3); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg = 1.8 m + RowFactory.create("patient-2", null), // 500 mg = 1.8 m + RowFactory.create("patient-3", null), // 500 mg = 1.8 m + RowFactory.create("patient-4", null), // 650 mg = 1.8 m + RowFactory.create("patient-5", null), // {} = 1.8 m + RowFactory.create("patient-6", null), // 500 mg = 1.8 m + RowFactory.create("patient-7", null), // 30 d = 1.8 m + RowFactory.create("patient-8", null), // 60 s = 1.8 m + RowFactory.create("patient-9", null) // 1000 ms = 1.8 m + ); + + input = new OperatorInput(parserContext, ucumQuantityLiteral1, ucumQuantityLiteral1); + result = equalityOperator.invoke(input); + + final Dataset allTrue = new DatasetBuilder(spark) + .withIdsAndValue(true, + List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", + "patient-7", "patient-8", "patient-9")) + .build(); + assertThat(result).selectOrderedResult().hasRows(allTrue); + } + + @Test + void ucumLiteralNotEquals() { + OperatorInput input = new OperatorInput(parserContext, left, ucumQuantityLiteral1); + final Operator equalityOperator = Operator.getInstance("!="); + FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // 500 mg != 500 mg + RowFactory.create("patient-2", false), // 500 mg != 500 mg + RowFactory.create("patient-3", false), // 500 mg != 500 mg + RowFactory.create("patient-4", true), // 650 mg != 500 mg + RowFactory.create("patient-5", null), // {} != 500 mg + RowFactory.create("patient-6", false), // 500 mg != 500 mg + RowFactory.create("patient-7", null), // 30 d != 500 mg + RowFactory.create("patient-8", null), // 60 s != 500 mg + RowFactory.create("patient-9", null) // 1000 ms != 500 mg + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteral2); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // 500 mg != 0.5 mg + RowFactory.create("patient-2", false), // 500 mg != 0.5 mg + RowFactory.create("patient-3", false), // 500 mg != 0.5 mg + RowFactory.create("patient-4", true), // 650 mg != 0.5 mg + RowFactory.create("patient-5", null), // {} != 0.5 mg + RowFactory.create("patient-6", false), // 500 mg != 0.5 mg + RowFactory.create("patient-7", null), // 30 d != 0.5 mg + RowFactory.create("patient-8", null), // 60 s != 0.5 mg + RowFactory.create("patient-9", null) // 1000 ms != 0.5 mg + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteral3); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg != 1.8 m + RowFactory.create("patient-2", null), // 500 mg != 1.8 m + RowFactory.create("patient-3", null), // 500 mg != 1.8 m + RowFactory.create("patient-4", null), // 650 mg != 1.8 m + RowFactory.create("patient-5", null), // {} != 1.8 m + RowFactory.create("patient-6", null), // 500 mg != 1.8 m + RowFactory.create("patient-7", null), // 30 d != 1.8 m + RowFactory.create("patient-8", null), // 60 s != 1.8 m + RowFactory.create("patient-9", null) // 1000 ms != 1.8 m + ); + + input = new OperatorInput(parserContext, ucumQuantityLiteral1, ucumQuantityLiteral1); + result = equalityOperator.invoke(input); + + final Dataset allFalse = new DatasetBuilder(spark) + .withIdsAndValue(false, + List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", + "patient-7", "patient-8", "patient-9")) + .build(); + assertThat(result).selectOrderedResult().hasRows(allFalse); + } + + @Test + void calendarLiteralEquals() { + OperatorInput input = new OperatorInput(parserContext, left, calendarDurationLiteral1); + final Operator equalityOperator = Operator.getInstance("="); + FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg = 30 days + RowFactory.create("patient-2", null), // 500 mg = 30 days + RowFactory.create("patient-3", null), // 500 mg = 30 days + RowFactory.create("patient-4", null), // 650 mg = 30 days + RowFactory.create("patient-5", null), // {} = 30 days + RowFactory.create("patient-6", null), // 500 mg = 30 days + RowFactory.create("patient-7", true), // 30 d = 30 days + RowFactory.create("patient-8", null), // 60 s = 30 days + RowFactory.create("patient-9", null) // 1000 ms = 30 days + ); + + input = new OperatorInput(parserContext, left, calendarDurationLiteral2); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg = 60 s + RowFactory.create("patient-2", null), // 500 mg = 60 s + RowFactory.create("patient-3", null), // 500 mg = 60 s + RowFactory.create("patient-4", null), // 650 mg = 60 s + RowFactory.create("patient-5", null), // {} = 60 s + RowFactory.create("patient-6", null), // 500 mg = 60 s + RowFactory.create("patient-7", null), // 30 d = 60 s + RowFactory.create("patient-8", true), // 60 s = 60 s + RowFactory.create("patient-9", false) // 1000 ms = 60 s + ); + + input = new OperatorInput(parserContext, left, calendarDurationLiteral3); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg = 1000 ms + RowFactory.create("patient-2", null), // 500 mg = 1000 ms + RowFactory.create("patient-3", null), // 500 mg = 1000 ms + RowFactory.create("patient-4", null), // 650 mg = 1000 ms + RowFactory.create("patient-5", null), // {} = 1000 ms + RowFactory.create("patient-6", null), // 500 mg = 1000 ms + RowFactory.create("patient-7", null), // 30 d = 1000 ms + RowFactory.create("patient-8", false), // 60 s = 1000 ms + RowFactory.create("patient-9", true) // 1000 ms = 1000 ms + ); + + input = new OperatorInput(parserContext, calendarDurationLiteral1, calendarDurationLiteral1); + result = equalityOperator.invoke(input); + + final Dataset allTrue = new DatasetBuilder(spark) + .withIdsAndValue(true, + List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", + "patient-7", "patient-8", "patient-9")) + .build(); + assertThat(result).selectOrderedResult().hasRows(allTrue); + } + + @Test + void calendarLiteralNotEquals() { + OperatorInput input = new OperatorInput(parserContext, left, calendarDurationLiteral1); + final Operator equalityOperator = Operator.getInstance("!="); + FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg != 30 days + RowFactory.create("patient-2", null), // 500 mg != 30 days + RowFactory.create("patient-3", null), // 500 mg != 30 days + RowFactory.create("patient-4", null), // 650 mg != 30 days + RowFactory.create("patient-5", null), // {} != 30 days + RowFactory.create("patient-6", null), // 500 mg != 30 days + RowFactory.create("patient-7", false), // 30 d != 30 days + RowFactory.create("patient-8", null), // 60 s != 30 days + RowFactory.create("patient-9", null) // 1000 ms != 30 days + ); + + input = new OperatorInput(parserContext, left, calendarDurationLiteral2); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg != 60 s + RowFactory.create("patient-2", null), // 500 mg != 60 s + RowFactory.create("patient-3", null), // 500 mg != 60 s + RowFactory.create("patient-4", null), // 650 mg != 60 s + RowFactory.create("patient-5", null), // {} != 60 s + RowFactory.create("patient-6", null), // 500 mg != 60 s + RowFactory.create("patient-7", null), // 30 d != 60 s + RowFactory.create("patient-8", false), // 60 s != 60 s + RowFactory.create("patient-9", true) // 1000 ms != 60 s + ); + + input = new OperatorInput(parserContext, left, calendarDurationLiteral3); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg != 1000 ms + RowFactory.create("patient-2", null), // 500 mg != 1000 ms + RowFactory.create("patient-3", null), // 500 mg != 1000 ms + RowFactory.create("patient-4", null), // 650 mg != 1000 ms + RowFactory.create("patient-5", null), // {} != 1000 ms + RowFactory.create("patient-6", null), // 500 mg != 1000 ms + RowFactory.create("patient-7", null), // 30 d != 1000 ms + RowFactory.create("patient-8", true), // 60 s != 1000 ms + RowFactory.create("patient-9", false) // 1000 ms != 1000 ms + ); + + input = new OperatorInput(parserContext, calendarDurationLiteral1, calendarDurationLiteral1); + result = equalityOperator.invoke(input); + + final Dataset allFalse = new DatasetBuilder(spark) + .withIdsAndValue(false, + List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", + "patient-7", "patient-8", "patient-9")) + .build(); + assertThat(result).selectOrderedResult().hasRows(allFalse); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 7ebaf4a751..fefa01a916 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -9,6 +9,7 @@ import static au.csiro.pathling.utilities.Preconditions.checkNotNull; import static org.apache.spark.sql.functions.col; +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import java.util.Collections; import java.util.List; @@ -21,9 +22,14 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; -import org.apache.spark.sql.types.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Quantity; import scala.collection.JavaConverters; import scala.collection.mutable.Buffer; @@ -94,6 +100,18 @@ public static StructType referenceStructType() { return new StructType(new StructField[]{id, coding, text}); } + @Nonnull + public static StructType quantityStructType() { + final Metadata metadata = new MetadataBuilder().build(); + final StructField id = new StructField("id", DataTypes.StringType, true, metadata); + final StructField value = new StructField("value", DataTypes.createDecimalType( + DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); + final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); + final StructField system = new StructField("system", DataTypes.StringType, true, metadata); + final StructField code = new StructField("code", DataTypes.StringType, true, metadata); + return new StructType(new StructField[]{id, value, unit, system, code}); + } + @Nonnull public static Row rowFromCoding(@Nonnull final Coding coding) { return new GenericRowWithSchema( @@ -133,6 +151,14 @@ public static Row rowFromCodeableConcept(@Nonnull final CodeableConcept codeable codeableConceptStructType()); } + @Nonnull + public static Row rowFromQuantity(@Nonnull final Quantity quantity) { + return new GenericRowWithSchema( + new Object[]{quantity.getId(), quantity.getValue(), quantity.getUnit(), + quantity.getSystem(), quantity.getCode()}, + quantityStructType()); + } + @Value public static class IdAndValueColumns { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java index 47ed4c4bac..233f0c69c3 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java @@ -43,6 +43,7 @@ public abstract class TestHelpers { public static final String LOINC_URL = "http://loinc.org"; public static final String SNOMED_URL = "http://snomed.info/sct"; + public static final String UCUM_URL = "http://unitsofmeasure.org/"; public static final String PARQUET_PATH = "src/test/resources/test-data/parquet"; public static final MediaType FHIR_MEDIA_TYPE = new MediaType("application", "fhir+json"); From c9aa2bf1e7af6da28d309c6848bf593079e2ea31 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 5 Apr 2022 10:48:29 +1000 Subject: [PATCH 04/83] Add tests for comparison and math operators --- .../ComparisonOperatorQuantityTest.java | 251 ++++++++++++++++++ .../fhirpath/operator/DateArithmeticTest.java | 11 + .../operator/MathOperatorQuantityTest.java | 175 ++++++++++++ .../pathling/test/helpers/SparkHelpers.java | 11 + 4 files changed, 448 insertions(+) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java new file mode 100644 index 0000000000..f25df2b503 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java @@ -0,0 +1,251 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static au.csiro.pathling.test.helpers.SparkHelpers.quantityStructType; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; +import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.TestHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.Collections; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.hl7.fhir.r4.model.Quantity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +@Tag("UnitTest") +public class ComparisonOperatorQuantityTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + @Autowired + UcumService ucumService; + + static final String ID_ALIAS = "_abc123"; + + FhirPath left; + FhirPath right; + ParserContext parserContext; + UcumQuantityLiteralPath ucumQuantityLiteral1; + UcumQuantityLiteralPath ucumQuantityLiteral2; + UcumQuantityLiteralPath ucumQuantityLiteral3; + CalendarDurationLiteralPath calendarDurationLiteral1; + CalendarDurationLiteralPath calendarDurationLiteral2; + CalendarDurationLiteralPath calendarDurationLiteral3; + + @BeforeEach + void setUp() { + final Quantity quantity1 = new Quantity(); + quantity1.setValue(500); + quantity1.setUnit("mg"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("mg"); + + final Quantity quantity2 = new Quantity(); + quantity2.setValue(0.5); + quantity2.setUnit("g"); + quantity2.setSystem(TestHelpers.UCUM_URL); + quantity2.setCode("g"); + + final Quantity quantity3 = new Quantity(); + quantity3.setValue(1.8); + quantity3.setUnit("m"); + quantity3.setSystem(TestHelpers.UCUM_URL); + quantity3.setCode("m"); + + final Quantity quantity4 = new Quantity(); + quantity4.setValue(0.5); + quantity4.setUnit("g"); + quantity4.setSystem(TestHelpers.SNOMED_URL); + quantity4.setCode("258682000"); + + final Quantity quantity5 = new Quantity(); + quantity1.setValue(650); + quantity1.setUnit("mg"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("mg"); + + final Quantity quantity6 = new Quantity(); + quantity1.setValue(30); + quantity1.setUnit("d"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("d"); + + final Quantity quantity7 = new Quantity(); + quantity1.setValue(60); + quantity1.setUnit("s"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("s"); + + final Quantity quantity8 = new Quantity(); + quantity1.setValue(1000); + quantity1.setUnit("ms"); + quantity1.setSystem(TestHelpers.UCUM_URL); + quantity1.setCode("ms"); + + final Quantity quantity9 = new Quantity(); + quantity2.setValue(0.2); + quantity2.setUnit("g"); + quantity2.setSystem(TestHelpers.UCUM_URL); + quantity2.setCode("g"); + + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-2", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-3", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-4", rowFromQuantity(quantity5)) // 650 mg + .withRow("patient-5", null) + .withRow("patient-6", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-10", rowFromQuantity(quantity9)) // 0.2 g + .buildWithStructValue(); + left = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-2", rowFromQuantity(quantity2)) // 0.5 g + .withRow("patient-3", rowFromQuantity(quantity3)) // 1.8 m + .withRow("patient-4", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-5", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-6", null) + .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-10", rowFromQuantity(quantity1)) // 500 mg + .buildWithStructValue(); + right = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(rightDataset) + .idAndValueColumns() + .build(); + + ucumQuantityLiteral1 = UcumQuantityLiteralPath.fromString("500 'mg'", left, ucumService); + ucumQuantityLiteral2 = UcumQuantityLiteralPath.fromString("0.5 'g'", left, ucumService); + ucumQuantityLiteral3 = UcumQuantityLiteralPath.fromString("1.8 'm'", left, ucumService); + calendarDurationLiteral1 = CalendarDurationLiteralPath.fromString("30 days", left); + calendarDurationLiteral2 = CalendarDurationLiteralPath.fromString("60 seconds", left); + calendarDurationLiteral3 = CalendarDurationLiteralPath.fromString("1000 milliseconds", left); + + parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + } + + @Test + void lessThan() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance("<"); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // 500 mg < 500 mg + RowFactory.create("patient-2", false), // 500 mg < 0.5 g + RowFactory.create("patient-3", null), // 500 mg < 1.8 m + RowFactory.create("patient-4", false), // 650 mg < 500 mg + RowFactory.create("patient-5", null), // {} < 500 mg + RowFactory.create("patient-6", null), // 500 mg < {} + RowFactory.create("patient-7", false), // 30 d < 30 d + RowFactory.create("patient-8", false), // 60 s < 60 s + RowFactory.create("patient-9", false), // 1000 ms < 1000 ms + RowFactory.create("patient-10", true) // 0.2 g < 500 mg + ); + } + + @Test + void lessThanOrEqualTo() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance("<="); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // 500 mg <= 500 mg + RowFactory.create("patient-2", true), // 500 mg <= 0.5 g + RowFactory.create("patient-3", null), // 500 mg <= 1.8 m + RowFactory.create("patient-4", false), // 650 mg <= 500 mg + RowFactory.create("patient-5", null), // {} <= 500 mg + RowFactory.create("patient-6", null), // 500 mg <= {} + RowFactory.create("patient-7", true), // 30 d <= 30 d + RowFactory.create("patient-8", true), // 60 s <= 60 s + RowFactory.create("patient-9", true), // 1000 ms <= 1000 ms + RowFactory.create("patient-10", true) // 0.2 g <= 500 mg + ); + } + + @Test + void greaterThanOrEqualTo() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance(">="); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // 500 mg >= 500 mg + RowFactory.create("patient-2", true), // 500 mg >= 0.5 g + RowFactory.create("patient-3", null), // 500 mg >= 1.8 m + RowFactory.create("patient-4", true), // 650 mg >= 500 mg + RowFactory.create("patient-5", null), // {} >= 500 mg + RowFactory.create("patient-6", null), // 500 mg >= {} + RowFactory.create("patient-7", true), // 30 d >= 30 d + RowFactory.create("patient-8", true), // 60 s >= 60 s + RowFactory.create("patient-9", true), // 1000 ms >= 1000 ms + RowFactory.create("patient-10", false) // 0.2 g >= 500 mg + ); + } + + @Test + void greaterThan() { + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance(">"); + final FhirPath result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // 500 mg > 500 mg + RowFactory.create("patient-2", false), // 500 mg > 0.5 g + RowFactory.create("patient-3", null), // 500 mg > 1.8 m + RowFactory.create("patient-4", true), // 650 mg > 500 mg + RowFactory.create("patient-5", null), // {} > 500 mg + RowFactory.create("patient-6", null), // 500 mg > {} + RowFactory.create("patient-7", false), // 30 d > 30 d + RowFactory.create("patient-8", false), // 60 s > 60 s + RowFactory.create("patient-9", false), // 1000 ms > 1000 ms + RowFactory.create("patient-10", false) // 0.2 g > 500 mg + ); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java new file mode 100644 index 0000000000..74f5ea4e25 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -0,0 +1,11 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +public class DateArithmeticTest { + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java new file mode 100644 index 0000000000..6b35b67959 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java @@ -0,0 +1,175 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static au.csiro.pathling.test.helpers.SparkHelpers.quantityStructType; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowForUcumQuantity; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.TestHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Value; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.hl7.fhir.r4.model.Quantity; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +@TestInstance(Lifecycle.PER_CLASS) +@Tag("UnitTest") +public class MathOperatorQuantityTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final List OPERATORS = List.of("+", "-", "*", "/", "mod"); + static final String ID_ALIAS = "_abc123"; + + @Value + static class TestParameters { + + @Nonnull + String name; + + @Nonnull + FhirPath left; + + @Nonnull + FhirPath right; + + @Nonnull + ParserContext context; + + @Nonnull + Operator operator; + + @Nonnull + Dataset expectedResult; + + @Override + public String toString() { + return name; + } + + } + + @Nonnull + Stream parameters() { + final Collection parameters = new ArrayList<>(); + for (final String operator : OPERATORS) { + final String name = "Quantity " + operator + " Quantity"; + final FhirPath left = buildQuantityExpression(true); + final FhirPath right = buildQuantityExpression(false); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + parameters.add(new TestParameters(name, left, right, context, Operator.getInstance(operator), + expectedResult(operator))); + } + return parameters.stream(); + } + + Dataset expectedResult(@Nonnull final String operator) { + final Row result; + switch (operator) { + case "+": + result = rowForUcumQuantity(1650.0, "m"); + break; + case "-": + result = rowForUcumQuantity(-1350.0, "m"); + break; + case "*": + result = rowForUcumQuantity(225000.0, "m"); + break; + case "/": + case "mod": + result = rowForUcumQuantity(0.1, "m"); + break; + default: + result = null; + } + return new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", result) + .withRow("patient-2", null) + .withRow("patient-3", null) + .withRow("patient-4", null) + .withRow("patient-5", null) + .withRow("patient-6", null) + .buildWithStructValue(); + } + + @Nonnull + FhirPath buildQuantityExpression(@Nonnull final boolean leftOperand) { + final Quantity nonUcumQuantity = new Quantity(); + nonUcumQuantity.setValue(15); + nonUcumQuantity.setUnit("mSv"); + nonUcumQuantity.setSystem(TestHelpers.SNOMED_URL); + nonUcumQuantity.setCode("282250007"); + + final Dataset dataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", leftOperand + ? rowForUcumQuantity(150.0, "m") + : rowForUcumQuantity(1.5, "km")) + .withRow("patient-2", leftOperand + ? rowForUcumQuantity(7.7, "mSv") + : rowForUcumQuantity(1.5, "h")) // Not comparable + .withRow("patient-3", leftOperand + ? rowForUcumQuantity(7.7, "mSv") + : rowFromQuantity(nonUcumQuantity)) // Not comparable + .withRow("patient-4", leftOperand + ? null + : rowForUcumQuantity(1.5, "h")) + .withRow("patient-5", leftOperand + ? rowForUcumQuantity(7.7, "mSv") + : null) + .withRow("patient-6", null) + .buildWithStructValue(); + return new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .dataset(dataset) + .idAndValueColumns() + .singular(true) + .build(); + } + + @ParameterizedTest + @MethodSource("parameters") + void test(@Nonnull final TestParameters parameters) { + final OperatorInput input = new OperatorInput(parameters.getContext(), + parameters.getLeft(), parameters.getRight()); + final FhirPath result = parameters.getOperator().invoke(input); + assertThat(result).selectOrderedResult().hasRows(parameters.getExpectedResult()); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index fefa01a916..6dbf20c3eb 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -11,6 +11,7 @@ import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; +import java.math.BigDecimal; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -159,6 +160,16 @@ public static Row rowFromQuantity(@Nonnull final Quantity quantity) { quantityStructType()); } + @Nonnull + public static Row rowForUcumQuantity(@Nonnull final Double value, @Nonnull final String unit) { + final Quantity quantity = new Quantity(); + quantity.setValue(new BigDecimal(value)); + quantity.setUnit(unit); + quantity.setSystem(TestHelpers.UCUM_URL); + quantity.setCode(unit); + return rowFromQuantity(quantity); + } + @Value public static class IdAndValueColumns { From 495aa30bf3695ed4d02634927227365a19b63355 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 6 Apr 2022 16:21:51 +1000 Subject: [PATCH 05/83] Add tests for date arithmetic --- .../literal/CalendarDurationLiteralPath.java | 1 + .../fhirpath/operator/DateArithmeticTest.java | 821 ++++++++++++++++++ 2 files changed, 822 insertions(+) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java index 1f74ac64f9..595fd3f02c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java @@ -23,6 +23,7 @@ protected CalendarDurationLiteralPath( super(dataset, idColumn, literalValue); } + @Nonnull public static CalendarDurationLiteralPath fromString(final String s, final FhirPath left) { return null; } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index 74f5ea4e25..44bdc26713 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -6,6 +6,827 @@ package au.csiro.pathling.fhirpath.operator; +import static au.csiro.pathling.test.assertions.Assertions.assertThat; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import ca.uhn.fhir.context.FhirContext; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Value; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +@TestInstance(Lifecycle.PER_CLASS) +@Tag("UnitTest") public class DateArithmeticTest { + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final String ID_ALIAS = "_abc123"; + static final List OPERATORS = List.of("+", "-"); + + @Value + static class TestParameters { + + @Nonnull + String name; + + @Nonnull + FhirPath left; + + @Nonnull + FhirPath right; + + @Nonnull + ParserContext context; + + @Nonnull + Operator operator; + + @Nonnull + Dataset expectedResult; + + @Override + public String toString() { + return name; + } + + } + + @Nonnull + Stream parameters() throws ParseException { + final List parameters = new ArrayList<>(); + + final Dataset dateTimeDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T13:28:17-05:00") + .withRow("patient-2", "2017-01-01T00:00:00.000Z") + .withRow("patient-3", "2025-06-21T00:15:00+10:00") + .build(); + final ElementPath dateTimePath = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATETIME) + .dataset(dateTimeDataset) + .idAndValueColumns() + .singular(true) + .build(); + + final Dataset dateDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07") + .withRow("patient-2", "2017-01-01") + .withRow("patient-3", "2025-06-21") + .build(); + final ElementPath datePath = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(dateDataset) + .idAndValueColumns() + .singular(true) + .build(); + + final DateTimeLiteralPath dateTimeLiteral = DateTimeLiteralPath.fromString( + "@2015-02-07T18:28:17.000Z", dateTimePath); + final DateLiteralPath dateLiteral = DateLiteralPath.fromString("@2015-02-07", datePath); + + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(dateTimePath.getIdColumn())) + .build(); + + parameters.addAll(dateTimeAddition(dateTimePath, context)); + parameters.addAll(dateTimeSubtraction(dateTimePath, context)); + parameters.addAll(dateAddition(datePath, context)); + parameters.addAll(dateSubtraction(datePath, context)); + parameters.addAll(dateTimeLiteralAddition(datePath, context)); + parameters.addAll(dateTimeLiteralSubtraction(datePath, context)); + parameters.addAll(dateLiteralAddition(datePath, context)); + parameters.addAll(dateLiteralSubtraction(datePath, context)); + + return parameters.stream(); + } + + Collection dateTimeAddition( + final FhirPath dateTimePath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("DateTime + 10 years", dateTimePath, + CalendarDurationLiteralPath.fromString("10 years", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2025-02-07T18:28:17.000Z") + .withRow("patient-2", "2027-01-01T00:00:00.000Z") + .withRow("patient-3", "2035-06-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 9 months", dateTimePath, + CalendarDurationLiteralPath.fromString("9 months", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-11-07T18:28:17.000Z") + .withRow("patient-2", "2017-10-01T00:00:00.000Z") + .withRow("patient-3", "2026-03-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 2 weeks", dateTimePath, + CalendarDurationLiteralPath.fromString("2 weeks", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-21T18:28:17.000Z") + .withRow("patient-2", "2017-01-15T00:00:00.000Z") + .withRow("patient-3", "2025-07-04T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 30 days", dateTimePath, + CalendarDurationLiteralPath.fromString("30 days", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-03-09T18:28:17.000Z") + .withRow("patient-2", "2017-01-31T00:00:00.000Z") + .withRow("patient-3", "2025-07-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 12 hours", dateTimePath, + CalendarDurationLiteralPath.fromString("12 hours", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-08T06:28:17.000Z") + .withRow("patient-2", "2017-01-01T12:00:00.000Z") + .withRow("patient-3", "2025-06-21T12:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 30 minutes", dateTimePath, + CalendarDurationLiteralPath.fromString("30 minutes", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:58:17.000Z") + .withRow("patient-2", "2017-01-01T00:30:00.000Z") + .withRow("patient-3", "2025-06-20T14:45:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 10 seconds", dateTimePath, + CalendarDurationLiteralPath.fromString("10 seconds", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:27.000Z") + .withRow("patient-2", "2017-01-01T00:00:10.000Z") + .withRow("patient-3", "2025-06-20T14:15:10.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime + 300 milliseconds", dateTimePath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300Z") + .withRow("patient-2", "2017-01-01T00:00:00.300Z") + .withRow("patient-3", "2025-06-20T14:15:00.300Z") + .build()) + ); + return parameters; + } + + Collection dateTimeSubtraction( + final FhirPath dateTimePath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("DateTime - 10 years", dateTimePath, + CalendarDurationLiteralPath.fromString("10 years", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2005-02-07T18:28:17.000Z") + .withRow("patient-2", "2007-01-01T00:00:00.000Z") + .withRow("patient-3", "2015-06-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 9 months", dateTimePath, + CalendarDurationLiteralPath.fromString("9 months", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2014-05-07T18:28:17.000Z") + .withRow("patient-2", "2016-04-01T00:00:00.000Z") + .withRow("patient-3", "2024-09-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 2 weeks", dateTimePath, + CalendarDurationLiteralPath.fromString("2 weeks", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-24T18:28:17.000Z") + .withRow("patient-2", "2016-12-18T00:00:00.000Z") + .withRow("patient-3", "2025-06-06T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 30 days", dateTimePath, + CalendarDurationLiteralPath.fromString("30 days", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-08T18:28:17.000Z") + .withRow("patient-2", "2016-12-02T00:00:00.000Z") + .withRow("patient-3", "2025-05-21T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 12 hours", dateTimePath, + CalendarDurationLiteralPath.fromString("12 hours", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T06:28:17.000Z") + .withRow("patient-2", "2016-12-31T12:00:00.000Z") + .withRow("patient-3", "2025-06-20T02:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 30 minutes", dateTimePath, + CalendarDurationLiteralPath.fromString("30 minutes", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T17:58:17.000Z") + .withRow("patient-2", "2016-12-31T23:30:00.000Z") + .withRow("patient-3", "2025-06-20T13:45:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 10 seconds", dateTimePath, + CalendarDurationLiteralPath.fromString("10 seconds", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:07.000Z") + .withRow("patient-2", "2016-12-31T23:59:50.000Z") + .withRow("patient-3", "2025-06-20T14:14:50.000Z") + .build()) + ); + + parameters.add(new TestParameters("DateTime - 300 milliseconds", dateTimePath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700Z") + .withRow("patient-2", "2016-12-31T23:59:59.700Z") + .withRow("patient-3", "2025-06-20T14:14:59.700Z") + .build()) + ); + return parameters; + } + + Collection dateAddition( + final FhirPath datePath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("Date + 10 years", datePath, + CalendarDurationLiteralPath.fromString("10 years", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2025-02-07T18:28:17.000Z") + .withRow("patient-2", "2027-01-01T00:00:00.000Z") + .withRow("patient-3", "2035-06-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 9 months", datePath, + CalendarDurationLiteralPath.fromString("9 months", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-11-07T18:28:17.000Z") + .withRow("patient-2", "2017-10-01T00:00:00.000Z") + .withRow("patient-3", "2026-03-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 2 weeks", datePath, + CalendarDurationLiteralPath.fromString("2 weeks", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-21T18:28:17.000Z") + .withRow("patient-2", "2017-01-15T00:00:00.000Z") + .withRow("patient-3", "2025-07-04T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 30 days", datePath, + CalendarDurationLiteralPath.fromString("30 days", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-03-09T18:28:17.000Z") + .withRow("patient-2", "2017-01-31T00:00:00.000Z") + .withRow("patient-3", "2025-07-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 12 hours", datePath, + CalendarDurationLiteralPath.fromString("12 hours", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-08T06:28:17.000Z") + .withRow("patient-2", "2017-01-01T12:00:00.000Z") + .withRow("patient-3", "2025-06-21T12:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 30 minutes", datePath, + CalendarDurationLiteralPath.fromString("30 minutes", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:58:17.000Z") + .withRow("patient-2", "2017-01-01T00:30:00.000Z") + .withRow("patient-3", "2025-06-20T14:45:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 10 seconds", datePath, + CalendarDurationLiteralPath.fromString("10 seconds", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:27.000Z") + .withRow("patient-2", "2017-01-01T00:00:10.000Z") + .withRow("patient-3", "2025-06-20T14:15:10.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date + 300 milliseconds", datePath, + CalendarDurationLiteralPath.fromString("300 milliseconds", datePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300Z") + .withRow("patient-2", "2017-01-01T00:00:00.300Z") + .withRow("patient-3", "2025-06-20T14:15:00.300Z") + .build()) + ); + return parameters; + } + + Collection dateSubtraction( + final FhirPath datePath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("Date - 10 years", datePath, + CalendarDurationLiteralPath.fromString("10 years", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2005-02-07T18:28:17.000Z") + .withRow("patient-2", "2007-01-01T00:00:00.000Z") + .withRow("patient-3", "2015-06-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 9 months", datePath, + CalendarDurationLiteralPath.fromString("9 months", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2014-05-07T18:28:17.000Z") + .withRow("patient-2", "2016-04-01T00:00:00.000Z") + .withRow("patient-3", "2024-09-20T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 2 weeks", datePath, + CalendarDurationLiteralPath.fromString("2 weeks", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-24T18:28:17.000Z") + .withRow("patient-2", "2016-12-18T00:00:00.000Z") + .withRow("patient-3", "2025-06-06T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 30 days", datePath, + CalendarDurationLiteralPath.fromString("30 days", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-08T18:28:17.000Z") + .withRow("patient-2", "2016-12-02T00:00:00.000Z") + .withRow("patient-3", "2025-05-21T14:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 12 hours", datePath, + CalendarDurationLiteralPath.fromString("12 hours", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T06:28:17.000Z") + .withRow("patient-2", "2016-12-31T12:00:00.000Z") + .withRow("patient-3", "2025-06-20T02:15:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 30 minutes", datePath, + CalendarDurationLiteralPath.fromString("30 minutes", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T17:58:17.000Z") + .withRow("patient-2", "2016-12-31T23:30:00.000Z") + .withRow("patient-3", "2025-06-20T13:45:00.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 10 seconds", datePath, + CalendarDurationLiteralPath.fromString("10 seconds", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:07.000Z") + .withRow("patient-2", "2016-12-31T23:59:50.000Z") + .withRow("patient-3", "2025-06-20T14:14:50.000Z") + .build()) + ); + + parameters.add(new TestParameters("Date - 300 milliseconds", datePath, + CalendarDurationLiteralPath.fromString("300 milliseconds", datePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700Z") + .withRow("patient-2", "2016-12-31T23:59:59.700Z") + .withRow("patient-3", "2025-06-20T14:14:59.700Z") + .build()) + ); + return parameters; + } + + Collection dateTimeLiteralAddition( + final FhirPath dateTimeLiteralPath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("10 years", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2025-02-07T18:28:17.000Z") + .withRow("patient-2", "2025-02-07T18:28:17.000Z") + .withRow("patient-3", "2025-02-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("9 months", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-11-07T18:28:17.000Z") + .withRow("patient-2", "2015-11-07T18:28:17.000Z") + .withRow("patient-3", "2015-11-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("2 weeks", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-21T18:28:17.000Z") + .withRow("patient-2", "2015-02-21T18:28:17.000Z") + .withRow("patient-3", "2015-02-21T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("30 days", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-03-09T18:28:17.000Z") + .withRow("patient-2", "2015-03-09T18:28:17.000Z") + .withRow("patient-3", "2015-03-09T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("12 hours", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-08T06:28:17.000Z") + .withRow("patient-2", "2015-02-08T06:28:17.000Z") + .withRow("patient-3", "2015-02-08T06:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("30 minutes", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:58:17.000Z") + .withRow("patient-2", "2015-02-07T18:58:17.000Z") + .withRow("patient-3", "2015-02-07T18:58:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("10 seconds", dateTimeLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:27.000Z") + .withRow("patient-2", "2015-02-07T18:28:27.000Z") + .withRow("patient-3", "2015-02-07T18:28:27.000Z") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimeLiteralPath), + context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300Z") + .withRow("patient-2", "2015-02-07T18:28:17.300Z") + .withRow("patient-3", "2015-02-07T18:28:17.300Z") + .build()) + ); + return parameters; + } + + Collection dateTimeLiteralSubtraction( + final FhirPath dateTimeLiteralPath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("10 years", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2005-02-07T18:28:17.000Z") + .withRow("patient-2", "2005-02-07T18:28:17.000Z") + .withRow("patient-3", "2005-02-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("9 months", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2014-05-07T18:28:17.000Z") + .withRow("patient-2", "2014-05-07T18:28:17.000Z") + .withRow("patient-3", "2014-05-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("2 weeks", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-24T18:28:17.000Z") + .withRow("patient-2", "2015-01-24T18:28:17.000Z") + .withRow("patient-3", "2015-01-24T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("30 days", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-08T18:28:17.000Z") + .withRow("patient-2", "2015-01-08T18:28:17.000Z") + .withRow("patient-3", "2015-01-08T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("12 hours", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T06:28:17.000Z") + .withRow("patient-2", "2015-02-07T06:28:17.000Z") + .withRow("patient-3", "2015-02-07T06:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("30 minutes", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T17:58:17.000Z") + .withRow("patient-2", "2015-02-07T17:58:17.000Z") + .withRow("patient-3", "2015-02-07T17:58:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("10 seconds", dateTimeLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:07.000Z") + .withRow("patient-2", "2015-02-07T18:28:07.000Z") + .withRow("patient-3", "2015-02-07T18:28:07.000Z") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateTimeLiteralPath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimeLiteralPath), + context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700Z") + .withRow("patient-2", "2015-02-07T18:28:16.700Z") + .withRow("patient-3", "2015-02-07T18:28:16.700Z") + .build()) + ); + return parameters; + } + + Collection dateLiteralAddition( + final FhirPath dateLiteralPath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateLiteralPath, + CalendarDurationLiteralPath.fromString("10 years", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2025-02-07T18:28:17.000Z") + .withRow("patient-2", "2025-02-07T18:28:17.000Z") + .withRow("patient-3", "2025-02-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateLiteralPath, + CalendarDurationLiteralPath.fromString("9 months", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-11-07T18:28:17.000Z") + .withRow("patient-2", "2015-11-07T18:28:17.000Z") + .withRow("patient-3", "2015-11-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateLiteralPath, + CalendarDurationLiteralPath.fromString("2 weeks", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-21T18:28:17.000Z") + .withRow("patient-2", "2015-02-21T18:28:17.000Z") + .withRow("patient-3", "2015-02-21T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateLiteralPath, + CalendarDurationLiteralPath.fromString("30 days", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-03-09T18:28:17.000Z") + .withRow("patient-2", "2015-03-09T18:28:17.000Z") + .withRow("patient-3", "2015-03-09T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateLiteralPath, + CalendarDurationLiteralPath.fromString("12 hours", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-08T06:28:17.000Z") + .withRow("patient-2", "2015-02-08T06:28:17.000Z") + .withRow("patient-3", "2015-02-08T06:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateLiteralPath, + CalendarDurationLiteralPath.fromString("30 minutes", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:58:17.000Z") + .withRow("patient-2", "2015-02-07T18:58:17.000Z") + .withRow("patient-3", "2015-02-07T18:58:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateLiteralPath, + CalendarDurationLiteralPath.fromString("10 seconds", dateLiteralPath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:27.000Z") + .withRow("patient-2", "2015-02-07T18:28:27.000Z") + .withRow("patient-3", "2015-02-07T18:28:27.000Z") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateLiteralPath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateLiteralPath), + context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300Z") + .withRow("patient-2", "2015-02-07T18:28:17.300Z") + .withRow("patient-3", "2015-02-07T18:28:17.300Z") + .build()) + ); + return parameters; + } + + Collection dateLiteralSubtraction( + final FhirPath dateLiteralPath, final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateLiteralPath, + CalendarDurationLiteralPath.fromString("10 years", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2005-02-07T18:28:17.000Z") + .withRow("patient-2", "2005-02-07T18:28:17.000Z") + .withRow("patient-3", "2005-02-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateLiteralPath, + CalendarDurationLiteralPath.fromString("9 months", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2014-05-07T18:28:17.000Z") + .withRow("patient-2", "2014-05-07T18:28:17.000Z") + .withRow("patient-3", "2014-05-07T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateLiteralPath, + CalendarDurationLiteralPath.fromString("2 weeks", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-24T18:28:17.000Z") + .withRow("patient-2", "2015-01-24T18:28:17.000Z") + .withRow("patient-3", "2015-01-24T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateLiteralPath, + CalendarDurationLiteralPath.fromString("30 days", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-01-08T18:28:17.000Z") + .withRow("patient-2", "2015-01-08T18:28:17.000Z") + .withRow("patient-3", "2015-01-08T18:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateLiteralPath, + CalendarDurationLiteralPath.fromString("12 hours", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T06:28:17.000Z") + .withRow("patient-2", "2015-02-07T06:28:17.000Z") + .withRow("patient-3", "2015-02-07T06:28:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateLiteralPath, + CalendarDurationLiteralPath.fromString("30 minutes", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T17:58:17.000Z") + .withRow("patient-2", "2015-02-07T17:58:17.000Z") + .withRow("patient-3", "2015-02-07T17:58:17.000Z") + .build()) + ); + + parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateLiteralPath, + CalendarDurationLiteralPath.fromString("10 seconds", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:07.000Z") + .withRow("patient-2", "2015-02-07T18:28:07.000Z") + .withRow("patient-3", "2015-02-07T18:28:07.000Z") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateLiteralPath, + CalendarDurationLiteralPath.fromString("300 milliseconds", dateLiteralPath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700Z") + .withRow("patient-2", "2015-02-07T18:28:16.700Z") + .withRow("patient-3", "2015-02-07T18:28:16.700Z") + .build()) + ); + return parameters; + } + + @ParameterizedTest + @MethodSource("parameters") + void test(@Nonnull final TestParameters parameters) { + final OperatorInput input = new OperatorInput(parameters.getContext(), + parameters.getLeft(), parameters.getRight()); + final FhirPath result = parameters.getOperator().invoke(input); + assertThat(result).selectOrderedResult().hasRows(parameters.getExpectedResult()); + } + } From ca01ca058937a2e13362bfebdefa7a835192a670 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 7 Apr 2022 14:41:05 +1000 Subject: [PATCH 06/83] Get EqualityOperatorQuantityTest passing --- .../pathling/encoders/FhirEncodersTest.java | 29 +++- .../au/csiro/pathling/encoders/TestData.java | 37 ++++- .../fhirpath/element/QuantityPath.java | 36 +++-- .../fhirpath/encoding/QuantityEncoding.java | 59 +++++++- .../literal/CalendarDurationLiteralPath.java | 47 ------- .../fhirpath/literal/LiteralPath.java | 4 +- ...eralPath.java => QuantityLiteralPath.java} | 82 ++++------- .../fhirpath/parser/LiteralTermVisitor.java | 28 ++-- .../java/au/csiro/pathling/spark/Spark.java | 9 +- .../terminology/ucum/ComparableQuantity.java | 97 +++++++++++++ .../terminology/ucum/QuantityEquality.java | 79 ----------- .../ComparisonOperatorQuantityTest.java | 28 ++-- .../fhirpath/operator/DateArithmeticTest.java | 131 +++++++++--------- .../EqualityOperatorQuantityTest.java | 117 ++++++++-------- .../pathling/test/UnitTestDependencies.java | 6 +- .../pathling/test/helpers/SparkHelpers.java | 18 ++- site/docs/fhirpath/operators.md | 6 +- 17 files changed, 446 insertions(+), 367 deletions(-) delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java rename fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/{UcumQuantityLiteralPath.java => QuantityLiteralPath.java} (66%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java index 927063778e..15dcfbb71a 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java @@ -28,10 +28,26 @@ import java.util.stream.Stream; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; +import org.apache.spark.sql.functions; import org.hl7.fhir.exceptions.FHIRException; -import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Annotation; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.Encounter; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.MedicationRequest; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; +import org.hl7.fhir.r4.model.Questionnaire; +import org.hl7.fhir.r4.model.QuestionnaireResponse; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -520,4 +536,13 @@ public void testNestedQuestionnaire() { .getItem(0).getField("linkId")) .collectAsList()); } + + @Test + public void testQuantityComparator() { + final QuantityComparator originalComparator = observation.getValueQuantity().getComparator(); + final String queriedComparator = observationsDataset.select("valueQuantity.comparator").head() + .getString(0); + + Assert.assertEquals(originalComparator.toCode(), queriedComparator); + } } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java index bfc15f6891..30802d415a 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java @@ -20,13 +20,37 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.Annotation; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.Condition; import org.hl7.fhir.r4.model.Condition.ConditionStageComponent; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.DecimalType; +import org.hl7.fhir.r4.model.Encounter; +import org.hl7.fhir.r4.model.Extension; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Medication; import org.hl7.fhir.r4.model.Medication.MedicationIngredientComponent; +import org.hl7.fhir.r4.model.MedicationRequest; +import org.hl7.fhir.r4.model.Meta; +import org.hl7.fhir.r4.model.Narrative; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Provenance; import org.hl7.fhir.r4.model.Provenance.ProvenanceEntityRole; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; +import org.hl7.fhir.r4.model.Questionnaire; import org.hl7.fhir.r4.model.Questionnaire.QuestionnaireItemComponent; +import org.hl7.fhir.r4.model.QuestionnaireResponse; import org.hl7.fhir.r4.model.QuestionnaireResponse.QuestionnaireResponseItemAnswerComponent; import org.hl7.fhir.r4.model.QuestionnaireResponse.QuestionnaireResponseItemComponent; +import org.hl7.fhir.r4.model.Range; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.StringType; import org.hl7.fhir.r4.model.codesystems.ConditionVerStatus; import org.hl7.fhir.utilities.xhtml.NodeType; import org.hl7.fhir.utilities.xhtml.XhtmlNode; @@ -128,6 +152,7 @@ public static Observation newObservation() { final Quantity quantity = new Quantity(); quantity.setValue(TEST_SMALL_DECIMAL); quantity.setUnit("mm[Hg]"); + quantity.setComparator(QuantityComparator.LESS_THAN); observation.setValue(quantity); observation.setIssued(TEST_DATE); @@ -394,9 +419,9 @@ public static Condition newConditionWithExtensions() { nestedExtension.addExtension(new Extension("uuid:nested", new StringType("nested"))); conditionWithExtension.setExtension(Arrays.asList( - new Extension("uuid:ext1", new StringType("ext1")), - new Extension("uuid:ext2", new IntegerType(2)), - nestedExtension + new Extension("uuid:ext1", new StringType("ext1")), + new Extension("uuid:ext2", new IntegerType(2)), + nestedExtension ) ); @@ -410,8 +435,8 @@ public static Condition newConditionWithExtensions() { identifier.setId("uuid:identifier1"); identifier.setExtension(Arrays.asList( - new Extension("uuid:ext10", new StringType("ext10")), - new Extension("uuid:ext11", new IntegerType(11)) + new Extension("uuid:ext10", new StringType("ext10")), + new Extension("uuid:ext11", new IntegerType(11)) ) ); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 76db401c51..0790782e33 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -6,12 +6,15 @@ package au.csiro.pathling.fhirpath.element; -import au.csiro.pathling.errors.InvalidUserInputError; +import static org.apache.spark.sql.functions.callUDF; +import static org.apache.spark.sql.functions.when; + import au.csiro.pathling.fhirpath.Comparable; +import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; -import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; -import au.csiro.pathling.terminology.ucum.QuantityEquality; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.terminology.ucum.ComparableQuantity; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.Function; @@ -19,7 +22,6 @@ import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; -import org.apache.spark.sql.functions; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; /** @@ -30,7 +32,7 @@ public class QuantityPath extends ElementPath implements Comparable { public static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet - .of(QuantityPath.class, UcumQuantityLiteralPath.class, NullLiteralPath.class); + .of(QuantityPath.class, QuantityLiteralPath.class, NullLiteralPath.class); protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Optional eidColumn, @@ -47,15 +49,21 @@ public Function getComparison(@Nonnull final ComparisonOpera return buildComparison(this, operation); } - public static Function buildComparison(@Nonnull final Comparable source, - final @Nonnull ComparisonOperation operation) { - if (operation == ComparisonOperation.EQUALS) { - return Comparable.buildComparison(source, - (l, r) -> functions.callUDF(QuantityEquality.FUNCTION_NAME, l, r)); - } else { - throw new InvalidUserInputError( - "Quantity type does not support comparison operator: " + operation); - } + @Nonnull + public static Function buildComparison(@Nonnull final FhirPath source, + @Nonnull final ComparisonOperation operation) { + return target -> { + final Column comparableSource = callUDF(ComparableQuantity.FUNCTION_NAME, + source.getValueColumn()); + final Column comparableTarget = callUDF(ComparableQuantity.FUNCTION_NAME, + target.getValueColumn()); + final Column sourceCode = comparableSource.getField("code"); + final Column targetCode = comparableTarget.getField("code"); + final Column sourceValue = comparableSource.getField("value"); + final Column targetValue = comparableTarget.getField("value"); + final Column compareValues = operation.getSparkFunction().apply(sourceValue, targetValue); + return when(sourceCode.equalTo(targetCode), compareValues).otherwise(null); + }; } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 00a32b3a0b..0b46af71e5 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -6,10 +6,20 @@ package au.csiro.pathling.fhirpath.encoding; +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.fhirpath.element.DecimalPath; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; @@ -20,21 +30,56 @@ public static Row encode(@Nullable final Quantity quantity) { if (quantity == null) { return null; } + final String comparator = Optional.ofNullable(quantity.getComparator()) + .map(QuantityComparator::toCode).orElse(null); return RowFactory.create(quantity.getId(), quantity.getValue(), - quantity.getComparator().toCode(), quantity.getUnit(), quantity.getSystem(), + DecimalPath.getDecimalType().scale(), comparator, quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */); } @Nonnull public static Quantity decode(@Nonnull final Row row) { final Quantity quantity = new Quantity(); - quantity.setId(row.getString(0)); - quantity.setValue(row.getDecimal(1)); - quantity.setComparator(QuantityComparator.fromCode(row.getString(2))); - quantity.setUnit(row.getString(3)); - quantity.setSystem(row.getString(4)); - quantity.setCode(row.getString(5)); + + Optional.ofNullable(row.getString(0)).ifPresent(quantity::setId); + + // The value gets converted to a BigDecimal, taking into account the scale that has been encoded + // alongside it. + final int scale = row.getInt(2); + final BigDecimal value = Optional.ofNullable(row.getDecimal(1)) + .map(bd -> bd.setScale(scale, RoundingMode.HALF_UP)) + .orElse(null); + quantity.setValue(value); + + // The comparator is encoded as a string code, we need to convert it back to an enum. + Optional.ofNullable(row.getString(3)) + .map(QuantityComparator::fromCode) + .ifPresent(quantity::setComparator); + + Optional.ofNullable(row.getString(4)).ifPresent(quantity::setUnit); + Optional.ofNullable(row.getString(5)).ifPresent(quantity::setSystem); + Optional.ofNullable(row.getString(6)).ifPresent(quantity::setCode); + return quantity; } + @Nonnull + public static StructType dataType() { + final Metadata metadata = new MetadataBuilder().build(); + final StructField id = new StructField("id", DataTypes.StringType, true, metadata); + final StructField value = new StructField("value", DataTypes.createDecimalType( + DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); + final StructField valueScale = new StructField("value_scale", DataTypes.IntegerType, true, + metadata); + final StructField comparator = new StructField("comparator", DataTypes.StringType, true, + metadata); + final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); + final StructField system = new StructField("system", DataTypes.StringType, true, metadata); + final StructField code = new StructField("code", DataTypes.StringType, true, metadata); + final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, + metadata); + return new StructType( + new StructField[]{id, value, valueScale, comparator, unit, system, code, fid}); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java deleted file mode 100644 index 595fd3f02c..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CalendarDurationLiteralPath.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.fhirpath.literal; - -import au.csiro.pathling.fhirpath.FhirPath; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.hl7.fhir.r4.model.Type; - -public class CalendarDurationLiteralPath extends LiteralPath implements Comparable { - - protected CalendarDurationLiteralPath( - @Nonnull final Dataset dataset, - @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { - super(dataset, idColumn, literalValue); - } - - @Nonnull - public static CalendarDurationLiteralPath fromString(final String s, final FhirPath left) { - return null; - } - - @Nonnull - @Override - public String getExpression() { - return null; - } - - @Nullable - @Override - public Object getJavaValue() { - return null; - } - - @Override - public int compareTo(@Nonnull final Object o) { - return 0; - } -} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java index 8f1ae98471..850d4d1124 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java @@ -55,7 +55,7 @@ public abstract class LiteralPath implements FhirPath { .put(FHIRDefinedType.INSTANT, DateTimeLiteralPath.class) .put(FHIRDefinedType.TIME, TimeLiteralPath.class) .put(FHIRDefinedType.CODING, CodingLiteralPath.class) - .put(FHIRDefinedType.QUANTITY, UcumQuantityLiteralPath.class) + .put(FHIRDefinedType.QUANTITY, QuantityLiteralPath.class) .build(); private static final Map, FHIRDefinedType> FHIRPATH_TYPE_TO_FHIR_TYPE = @@ -68,7 +68,7 @@ public abstract class LiteralPath implements FhirPath { .put(DateTimeLiteralPath.class, FHIRDefinedType.DATETIME) .put(TimeLiteralPath.class, FHIRDefinedType.TIME) .put(CodingLiteralPath.class, FHIRDefinedType.CODING) - .put(UcumQuantityLiteralPath.class, FHIRDefinedType.QUANTITY) + .put(QuantityLiteralPath.class, FHIRDefinedType.QUANTITY) .build(); @Getter diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java similarity index 66% rename from fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java rename to fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 61023a255d..11aa97a79e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/UcumQuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -15,9 +15,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.terminology.ucum.Ucum; -import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; -import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.regex.Matcher; @@ -38,34 +36,15 @@ * @author John Grimes */ @Getter -public class UcumQuantityLiteralPath extends LiteralPath implements Comparable { +public class QuantityLiteralPath extends LiteralPath implements Comparable { + + public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); - private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() - .put("year", "a") - .put("years", "a") - .put("month", "mo") - .put("months", "mo") - .put("week", "wk") - .put("weeks", "wk") - .put("day", "d") - .put("days", "d") - .put("hour", "h") - .put("hours", "h") - .put("minute", "min") - .put("minutes", "min") - .put("second", "s") - .put("seconds", "s") - .put("millisecond", "ms") - .put("milliseconds", "ms") - .build(); - - @SuppressWarnings("WeakerAccess") - protected UcumQuantityLiteralPath(@Nonnull final Dataset dataset, - @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + protected QuantityLiteralPath(@Nonnull final Dataset dataset, + @Nonnull final Column idColumn, @Nonnull final Type literalValue) { super(dataset, idColumn, literalValue); check(literalValue instanceof Quantity); } @@ -81,26 +60,12 @@ protected UcumQuantityLiteralPath(@Nonnull final Dataset dataset, * @throws IllegalArgumentException if the literal is malformed */ @Nonnull - public static UcumQuantityLiteralPath fromString(@Nonnull final String fhirPath, - @Nonnull final FhirPath context, final UcumService ucumService) - throws IllegalArgumentException { - final Matcher ucumMatcher = UCUM_PATTERN.matcher(fhirPath); - if (ucumMatcher.matches()) { - return fromUcumString(ucumMatcher, context, ucumService); - } else { - final Matcher calendarDurationMatcher = CALENDAR_DURATION_PATTERN.matcher(fhirPath); - if (calendarDurationMatcher.matches()) { - return fromCalendarDurationString(calendarDurationMatcher, context, ucumService); - } else { - throw new IllegalArgumentException("Quantity literal has invalid format: " + fhirPath); - } - } - - } - - @Nonnull - private static UcumQuantityLiteralPath fromUcumString(@Nonnull final Matcher matcher, + public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, @Nonnull final FhirPath context, @Nonnull final UcumService ucumService) { + final Matcher matcher = UCUM_PATTERN.matcher(fhirPath); + if (!matcher.matches()) { + throw new IllegalArgumentException("UCUM Quantity literal has invalid format: " + fhirPath); + } final String fullPath = matcher.group(0); final String value = matcher.group(1); final String rawUnit = matcher.group(2); @@ -121,20 +86,22 @@ private static UcumQuantityLiteralPath fromUcumString(@Nonnull final Matcher mat } @Nonnull - private static UcumQuantityLiteralPath fromCalendarDurationString(@Nonnull final Matcher matcher, - @Nonnull final FhirPath context, final UcumService ucumService) { + public static QuantityLiteralPath fromCalendarDurationString(@Nonnull final String fhirPath, + @Nonnull final FhirPath context) { + final Matcher matcher = CALENDAR_DURATION_PATTERN.matcher(fhirPath); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Calendar duration literal has invalid format: " + fhirPath); + } final String value = matcher.group(1); final String keyword = matcher.group(2); - if (!CALENDAR_DURATION_TO_UCUM.containsKey(keyword)) { - throw new IllegalArgumentException("Invalid calendar duration keyword: " + keyword); - } - - final String unit = CALENDAR_DURATION_TO_UCUM.get(keyword); - final BigDecimal decimalValue = getQuantityValue(value, context); - final String display = ucumService.getCommonDisplay(unit); + final Quantity quantity = new Quantity(); + quantity.setValue(new BigDecimal(value)); + quantity.setSystem(FHIRPATH_CALENDAR_DURATION_URI); + quantity.setCode(keyword); - return buildLiteralPath(decimalValue, unit, display, context); + return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); } private static BigDecimal getQuantityValue(final String value, final @Nonnull FhirPath context) { @@ -148,7 +115,7 @@ private static BigDecimal getQuantityValue(final String value, final @Nonnull Fh } @Nonnull - private static UcumQuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, + private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, final String unit, final String display, final @Nonnull FhirPath context) { final Quantity quantity = new Quantity(); quantity.setValue(decimalValue); @@ -156,7 +123,7 @@ private static UcumQuantityLiteralPath buildLiteralPath(final BigDecimal decimal quantity.setCode(unit); quantity.setUnit(display); - return new UcumQuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); + return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); } @Nonnull @@ -184,6 +151,7 @@ public Column buildValueColumn() { return struct( lit(value.getId()).as("id"), lit(value.getValue()).as("value"), + lit(value.getValue().scale()).as("value_scale"), lit(comparator.map(QuantityComparator::toCode).orElse(null)).as("comparator"), lit(value.getUnit()).as("unit"), lit(value.getSystem()).as("system"), diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java index 9372e4c868..9414c2f059 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java @@ -17,9 +17,9 @@ import au.csiro.pathling.fhirpath.literal.DecimalLiteralPath; import au.csiro.pathling.fhirpath.literal.IntegerLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.literal.StringLiteralPath; import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; -import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.generated.FhirPathBaseVisitor; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.BooleanLiteralContext; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.CodingLiteralContext; @@ -34,6 +34,7 @@ import java.text.ParseException; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.antlr.v4.runtime.tree.TerminalNode; import org.fhir.ucum.UcumException; /** @@ -147,15 +148,24 @@ public FhirPath visitNullLiteral(@Nullable final NullLiteralContext ctx) { public FhirPath visitQuantityLiteral(@Nullable final QuantityLiteralContext ctx) { checkNotNull(ctx); @Nullable final String number = ctx.quantity().NUMBER().getText(); - @Nullable final String unit = ctx.quantity().unit().getText(); checkNotNull(number); - checkNotNull(unit); - final String fhirPath = String.format("%s %s", number, unit); - try { - return UcumQuantityLiteralPath.fromString(fhirPath, - context.getThisContext().orElse(context.getInputContext()), Ucum.ucumEssenceService()); - } catch (final UcumException e) { - throw new RuntimeException(e); + + final FhirPath resultContext = this.context.getThisContext().orElse(context.getInputContext()); + @Nullable final TerminalNode ucumUnit = ctx.quantity().unit().STRING(); + + if (ucumUnit == null) { + // Create a calendar duration literal. + final String fhirPath = String.format("%s %s", number, ctx.quantity().unit().getText()); + return QuantityLiteralPath.fromCalendarDurationString(fhirPath, resultContext); + } else { + // Create a UCUM Quantity literal. + final String fhirPath = String.format("%s %s", number, ucumUnit.getText()); + try { + return QuantityLiteralPath.fromUcumString(fhirPath, resultContext, + Ucum.ucumEssenceService()); + } catch (final UcumException e) { + throw new RuntimeException(e); + } } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index d40ec126b1..841ea07f12 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -9,9 +9,10 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.Configuration.Storage.Aws; import au.csiro.pathling.async.SparkListener; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.sql.PathlingStrategy; import au.csiro.pathling.terminology.CodingToLiteral; -import au.csiro.pathling.terminology.ucum.QuantityEquality; +import au.csiro.pathling.terminology.ucum.ComparableQuantity; import java.util.Arrays; import java.util.Objects; import java.util.Optional; @@ -52,7 +53,7 @@ public class Spark { public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, - @Nonnull final QuantityEquality quantityEquality) { + @Nonnull final ComparableQuantity comparableQuantity) { log.debug("Creating Spark session"); resolveSparkConfiguration(environment); @@ -65,7 +66,9 @@ public static SparkSession build(@Nonnull final Configuration configuration, PathlingStrategy.setup(spark); spark.udf() .register(CodingToLiteral.FUNCTION_NAME, new CodingToLiteral(), DataTypes.StringType); - spark.udf().register(QuantityEquality.FUNCTION_NAME, quantityEquality, DataTypes.BooleanType); + spark.udf() + .register(ComparableQuantity.FUNCTION_NAME, comparableQuantity, + QuantityEncoding.dataType()); // Configure AWS driver and credentials. configureAwsDriver(configuration, spark); diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java new file mode 100644 index 0000000000..6638fe678e --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.terminology.ucum; + +import au.csiro.pathling.fhirpath.element.DecimalPath; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import com.google.common.collect.ImmutableMap; +import java.math.BigDecimal; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.api.java.UDF1; +import org.fhir.ucum.Decimal; +import org.fhir.ucum.Pair; +import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class ComparableQuantity implements UDF1 { + + @Nonnull + private final UcumService ucumService; + + private static final long serialVersionUID = 2317610455653365964L; + private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("second", "s") + .put("seconds", "s") + .put("millisecond", "ms") + .put("milliseconds", "ms") + .build(); + + /** + * The name of this function when used within SQL. + */ + public static final String FUNCTION_NAME = "comparable_quantity"; + + public ComparableQuantity(@Nonnull final UcumService ucumService) { + this.ucumService = ucumService; + } + + @Nullable + @Override + public Row call(@Nullable final Row row) throws Exception { + if (row == null) { + return null; + } + + final Quantity input = QuantityEncoding.decode(row); + + // If system and code are not populated, the Quantity will not be comparable. + if (!input.hasValue() || input.getSystem() == null || input.getCode() == null) { + return null; + } + + // If the Quantity has a comparator, it will not be comparable. + if (input.getComparator() != null && input.getComparator() != QuantityComparator.NULL) { + return null; + } + + final String resolvedCode; + if (input.getSystem().equals(Ucum.SYSTEM_URI)) { + resolvedCode = input.getCode(); + } else if (input.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI) && + CALENDAR_DURATION_TO_UCUM.containsKey(input.getCode())) { + resolvedCode = CALENDAR_DURATION_TO_UCUM.get(input.getCode()); + } else { + // If the Quantity is not UCUM or a comparable calendar duration, it is not comparable. + return null; + } + + // Use the UCUM library to get the canonical form of the Quantity. + final int maxPrecision = DecimalPath.getDecimalType().precision(); + final Decimal value = new Decimal(input.getValue().toPlainString(), maxPrecision); + final Pair canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); + + // Create a new Quantity object with the canonicalized result. + final Quantity result = new Quantity(); + result.setValue(new BigDecimal(canonical.getValue().asDecimal())); + result.setComparator(input.getComparator()); + result.setUnit(input.getUnit()); + result.setSystem(Ucum.SYSTEM_URI); + result.setCode(canonical.getCode()); + + return QuantityEncoding.encode(result); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java deleted file mode 100644 index 08fd22be19..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/QuantityEquality.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.terminology.ucum; - -import au.csiro.pathling.fhirpath.element.DecimalPath; -import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import java.util.Objects; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.api.java.UDF2; -import org.fhir.ucum.Decimal; -import org.fhir.ucum.Pair; -import org.fhir.ucum.UcumService; -import org.hl7.fhir.r4.model.Quantity; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class QuantityEquality implements UDF2 { - - @Nonnull - private final UcumService ucumService; - - private static final long serialVersionUID = -5960116379100195530L; - - /** - * The name of this function when used within SQL. - */ - public static final String FUNCTION_NAME = "quantity_equals"; - - public QuantityEquality(@Nonnull final UcumService ucumService) { - this.ucumService = ucumService; - } - - @Override - public Boolean call(final Row left, final Row right) throws Exception { - final Quantity leftQuantity = QuantityEncoding.decode(left); - final Quantity rightQuantity = QuantityEncoding.decode(right); - - final String leftSystem = leftQuantity.getSystem(); - final String rightSystem = rightQuantity.getSystem(); - final String leftCode = leftQuantity.getCode(); - final String rightCode = rightQuantity.getCode(); - if (!leftQuantity.hasValue() || !rightQuantity.hasValue() - || leftSystem == null || rightSystem == null - || leftCode == null || rightCode == null) { - return null; - } - - if (leftSystem.equals(rightSystem) && leftCode.equals(rightCode)) { - return leftQuantity.getValue().equals(rightQuantity.getValue()); - } - - final int maxPrecision = DecimalPath.getDecimalType().precision(); - final Decimal leftValue = new Decimal(leftQuantity.getValue().toPlainString(), maxPrecision); - final Decimal rightValue = new Decimal(rightQuantity.getValue().toPlainString(), maxPrecision); - if (leftSystem.equals(Ucum.SYSTEM_URI) && rightSystem.equals(Ucum.SYSTEM_URI) - && Objects.equals(leftQuantity.getComparator(), rightQuantity.getComparator()) - && ucumService.isComparable(leftCode, rightCode)) { - final Pair leftCanonical = ucumService.getCanonicalForm(new Pair(leftValue, leftCode)); - final Pair rightCanonical = ucumService.getCanonicalForm(new Pair(rightValue, rightCode)); - if (!leftCanonical.getCode().equals(rightCanonical.getCode())) { - log.warn("Encountered comparable canonical UCUM forms with different codes: {} and {}", - leftQuantity, rightQuantity); - return null; - } else { - return leftCanonical.getValue().equals(rightCanonical.getValue()); - } - } else { - return null; - } - } - -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java index f25df2b503..3244fb26b3 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java @@ -11,8 +11,7 @@ import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; import au.csiro.pathling.fhirpath.FhirPath; -import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; -import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.test.builders.DatasetBuilder; import au.csiro.pathling.test.builders.ElementPathBuilder; @@ -51,12 +50,12 @@ public class ComparisonOperatorQuantityTest { FhirPath left; FhirPath right; ParserContext parserContext; - UcumQuantityLiteralPath ucumQuantityLiteral1; - UcumQuantityLiteralPath ucumQuantityLiteral2; - UcumQuantityLiteralPath ucumQuantityLiteral3; - CalendarDurationLiteralPath calendarDurationLiteral1; - CalendarDurationLiteralPath calendarDurationLiteral2; - CalendarDurationLiteralPath calendarDurationLiteral3; + QuantityLiteralPath ucumQuantityLiteral1; + QuantityLiteralPath ucumQuantityLiteral2; + QuantityLiteralPath ucumQuantityLiteral3; + QuantityLiteralPath calendarDurationLiteral1; + QuantityLiteralPath calendarDurationLiteral2; + QuantityLiteralPath calendarDurationLiteral3; @BeforeEach void setUp() { @@ -156,12 +155,13 @@ void setUp() { .idAndValueColumns() .build(); - ucumQuantityLiteral1 = UcumQuantityLiteralPath.fromString("500 'mg'", left, ucumService); - ucumQuantityLiteral2 = UcumQuantityLiteralPath.fromString("0.5 'g'", left, ucumService); - ucumQuantityLiteral3 = UcumQuantityLiteralPath.fromString("1.8 'm'", left, ucumService); - calendarDurationLiteral1 = CalendarDurationLiteralPath.fromString("30 days", left); - calendarDurationLiteral2 = CalendarDurationLiteralPath.fromString("60 seconds", left); - calendarDurationLiteral3 = CalendarDurationLiteralPath.fromString("1000 milliseconds", left); + ucumQuantityLiteral1 = QuantityLiteralPath.fromUcumString("500 'mg'", left, ucumService); + ucumQuantityLiteral2 = QuantityLiteralPath.fromUcumString("0.5 'g'", left, ucumService); + ucumQuantityLiteral3 = QuantityLiteralPath.fromUcumString("1.8 'm'", left, ucumService); + calendarDurationLiteral1 = QuantityLiteralPath.fromCalendarDurationString("30 days", left); + calendarDurationLiteral2 = QuantityLiteralPath.fromCalendarDurationString("60 seconds", left); + calendarDurationLiteral3 = QuantityLiteralPath.fromCalendarDurationString("1000 milliseconds", + left); parserContext = new ParserContextBuilder(spark, fhirContext) .groupingColumns(Collections.singletonList(left.getIdColumn())) diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index 44bdc26713..cd8a578cfd 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -10,9 +10,9 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.element.ElementPath; -import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.test.builders.DatasetBuilder; import au.csiro.pathling.test.builders.ElementPathBuilder; @@ -137,7 +137,7 @@ Collection dateTimeAddition( final FhirPath dateTimePath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("DateTime + 10 years", dateTimePath, - CalendarDurationLiteralPath.fromString("10 years", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2025-02-07T18:28:17.000Z") @@ -147,7 +147,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 9 months", dateTimePath, - CalendarDurationLiteralPath.fromString("9 months", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-11-07T18:28:17.000Z") @@ -157,7 +157,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 2 weeks", dateTimePath, - CalendarDurationLiteralPath.fromString("2 weeks", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-21T18:28:17.000Z") @@ -167,7 +167,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 30 days", dateTimePath, - CalendarDurationLiteralPath.fromString("30 days", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-03-09T18:28:17.000Z") @@ -177,7 +177,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 12 hours", dateTimePath, - CalendarDurationLiteralPath.fromString("12 hours", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-08T06:28:17.000Z") @@ -187,7 +187,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 30 minutes", dateTimePath, - CalendarDurationLiteralPath.fromString("30 minutes", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:58:17.000Z") @@ -197,7 +197,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 10 seconds", dateTimePath, - CalendarDurationLiteralPath.fromString("10 seconds", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:27.000Z") @@ -207,7 +207,7 @@ Collection dateTimeAddition( ); parameters.add(new TestParameters("DateTime + 300 milliseconds", dateTimePath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:17.300Z") @@ -222,7 +222,7 @@ Collection dateTimeSubtraction( final FhirPath dateTimePath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("DateTime - 10 years", dateTimePath, - CalendarDurationLiteralPath.fromString("10 years", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2005-02-07T18:28:17.000Z") @@ -232,7 +232,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 9 months", dateTimePath, - CalendarDurationLiteralPath.fromString("9 months", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2014-05-07T18:28:17.000Z") @@ -242,7 +242,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 2 weeks", dateTimePath, - CalendarDurationLiteralPath.fromString("2 weeks", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-24T18:28:17.000Z") @@ -252,7 +252,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 30 days", dateTimePath, - CalendarDurationLiteralPath.fromString("30 days", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-08T18:28:17.000Z") @@ -262,7 +262,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 12 hours", dateTimePath, - CalendarDurationLiteralPath.fromString("12 hours", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T06:28:17.000Z") @@ -272,7 +272,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 30 minutes", dateTimePath, - CalendarDurationLiteralPath.fromString("30 minutes", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T17:58:17.000Z") @@ -282,7 +282,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 10 seconds", dateTimePath, - CalendarDurationLiteralPath.fromString("10 seconds", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:07.000Z") @@ -292,7 +292,7 @@ Collection dateTimeSubtraction( ); parameters.add(new TestParameters("DateTime - 300 milliseconds", dateTimePath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimePath), context, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:16.700Z") @@ -307,7 +307,7 @@ Collection dateAddition( final FhirPath datePath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("Date + 10 years", datePath, - CalendarDurationLiteralPath.fromString("10 years", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2025-02-07T18:28:17.000Z") @@ -317,7 +317,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 9 months", datePath, - CalendarDurationLiteralPath.fromString("9 months", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-11-07T18:28:17.000Z") @@ -327,7 +327,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 2 weeks", datePath, - CalendarDurationLiteralPath.fromString("2 weeks", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-21T18:28:17.000Z") @@ -337,7 +337,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 30 days", datePath, - CalendarDurationLiteralPath.fromString("30 days", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-03-09T18:28:17.000Z") @@ -347,7 +347,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 12 hours", datePath, - CalendarDurationLiteralPath.fromString("12 hours", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-08T06:28:17.000Z") @@ -357,7 +357,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 30 minutes", datePath, - CalendarDurationLiteralPath.fromString("30 minutes", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:58:17.000Z") @@ -367,7 +367,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 10 seconds", datePath, - CalendarDurationLiteralPath.fromString("10 seconds", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:27.000Z") @@ -377,7 +377,7 @@ Collection dateAddition( ); parameters.add(new TestParameters("Date + 300 milliseconds", datePath, - CalendarDurationLiteralPath.fromString("300 milliseconds", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:17.300Z") @@ -392,7 +392,7 @@ Collection dateSubtraction( final FhirPath datePath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("Date - 10 years", datePath, - CalendarDurationLiteralPath.fromString("10 years", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2005-02-07T18:28:17.000Z") @@ -402,7 +402,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 9 months", datePath, - CalendarDurationLiteralPath.fromString("9 months", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2014-05-07T18:28:17.000Z") @@ -412,7 +412,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 2 weeks", datePath, - CalendarDurationLiteralPath.fromString("2 weeks", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-24T18:28:17.000Z") @@ -422,7 +422,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 30 days", datePath, - CalendarDurationLiteralPath.fromString("30 days", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-08T18:28:17.000Z") @@ -432,7 +432,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 12 hours", datePath, - CalendarDurationLiteralPath.fromString("12 hours", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T06:28:17.000Z") @@ -442,7 +442,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 30 minutes", datePath, - CalendarDurationLiteralPath.fromString("30 minutes", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T17:58:17.000Z") @@ -452,7 +452,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 10 seconds", datePath, - CalendarDurationLiteralPath.fromString("10 seconds", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:07.000Z") @@ -462,7 +462,7 @@ Collection dateSubtraction( ); parameters.add(new TestParameters("Date - 300 milliseconds", datePath, - CalendarDurationLiteralPath.fromString("300 milliseconds", datePath), context, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:16.700Z") @@ -477,7 +477,7 @@ Collection dateTimeLiteralAddition( final FhirPath dateTimeLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("10 years", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2025-02-07T18:28:17.000Z") @@ -487,7 +487,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("9 months", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-11-07T18:28:17.000Z") @@ -497,7 +497,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("2 weeks", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-21T18:28:17.000Z") @@ -507,7 +507,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("30 days", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-03-09T18:28:17.000Z") @@ -517,7 +517,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("12 hours", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-08T06:28:17.000Z") @@ -527,7 +527,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("30 minutes", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:58:17.000Z") @@ -537,7 +537,7 @@ Collection dateTimeLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("10 seconds", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:27.000Z") @@ -548,7 +548,7 @@ Collection dateTimeLiteralAddition( parameters.add( new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimeLiteralPath), + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) @@ -564,7 +564,7 @@ Collection dateTimeLiteralSubtraction( final FhirPath dateTimeLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("10 years", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2005-02-07T18:28:17.000Z") @@ -574,7 +574,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("9 months", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2014-05-07T18:28:17.000Z") @@ -584,7 +584,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("2 weeks", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-24T18:28:17.000Z") @@ -594,7 +594,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("30 days", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-08T18:28:17.000Z") @@ -604,7 +604,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("12 hours", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T06:28:17.000Z") @@ -614,7 +614,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("30 minutes", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T17:58:17.000Z") @@ -624,7 +624,7 @@ Collection dateTimeLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("10 seconds", dateTimeLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:07.000Z") @@ -635,7 +635,7 @@ Collection dateTimeLiteralSubtraction( parameters.add( new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateTimeLiteralPath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateTimeLiteralPath), + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) @@ -651,7 +651,7 @@ Collection dateLiteralAddition( final FhirPath dateLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateLiteralPath, - CalendarDurationLiteralPath.fromString("10 years", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2025-02-07T18:28:17.000Z") @@ -661,7 +661,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateLiteralPath, - CalendarDurationLiteralPath.fromString("9 months", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-11-07T18:28:17.000Z") @@ -671,7 +671,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateLiteralPath, - CalendarDurationLiteralPath.fromString("2 weeks", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-21T18:28:17.000Z") @@ -681,7 +681,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateLiteralPath, - CalendarDurationLiteralPath.fromString("30 days", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-03-09T18:28:17.000Z") @@ -691,7 +691,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateLiteralPath, - CalendarDurationLiteralPath.fromString("12 hours", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-08T06:28:17.000Z") @@ -701,7 +701,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateLiteralPath, - CalendarDurationLiteralPath.fromString("30 minutes", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:58:17.000Z") @@ -711,7 +711,7 @@ Collection dateLiteralAddition( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateLiteralPath, - CalendarDurationLiteralPath.fromString("10 seconds", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:27.000Z") @@ -722,7 +722,7 @@ Collection dateLiteralAddition( parameters.add( new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateLiteralPath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateLiteralPath), + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) @@ -738,7 +738,7 @@ Collection dateLiteralSubtraction( final FhirPath dateLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateLiteralPath, - CalendarDurationLiteralPath.fromString("10 years", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 years", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2005-02-07T18:28:17.000Z") @@ -748,7 +748,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateLiteralPath, - CalendarDurationLiteralPath.fromString("9 months", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("9 months", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2014-05-07T18:28:17.000Z") @@ -758,7 +758,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateLiteralPath, - CalendarDurationLiteralPath.fromString("2 weeks", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-24T18:28:17.000Z") @@ -768,7 +768,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateLiteralPath, - CalendarDurationLiteralPath.fromString("30 days", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-01-08T18:28:17.000Z") @@ -778,7 +778,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateLiteralPath, - CalendarDurationLiteralPath.fromString("12 hours", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("12 hours", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T06:28:17.000Z") @@ -788,7 +788,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateLiteralPath, - CalendarDurationLiteralPath.fromString("30 minutes", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T17:58:17.000Z") @@ -798,7 +798,7 @@ Collection dateLiteralSubtraction( ); parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateLiteralPath, - CalendarDurationLiteralPath.fromString("10 seconds", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:07.000Z") @@ -809,7 +809,8 @@ Collection dateLiteralSubtraction( parameters.add( new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateLiteralPath, - CalendarDurationLiteralPath.fromString("300 milliseconds", dateLiteralPath), context, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateLiteralPath), + context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T18:28:16.700Z") diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java index f409c65dee..f185da85af 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java @@ -11,8 +11,7 @@ import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; import au.csiro.pathling.fhirpath.FhirPath; -import au.csiro.pathling.fhirpath.literal.CalendarDurationLiteralPath; -import au.csiro.pathling.fhirpath.literal.UcumQuantityLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.test.builders.DatasetBuilder; import au.csiro.pathling.test.builders.ElementPathBuilder; @@ -25,6 +24,7 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.Quantity; @@ -52,12 +52,12 @@ public class EqualityOperatorQuantityTest { FhirPath left; FhirPath right; ParserContext parserContext; - UcumQuantityLiteralPath ucumQuantityLiteral1; - UcumQuantityLiteralPath ucumQuantityLiteral2; - UcumQuantityLiteralPath ucumQuantityLiteral3; - CalendarDurationLiteralPath calendarDurationLiteral1; - CalendarDurationLiteralPath calendarDurationLiteral2; - CalendarDurationLiteralPath calendarDurationLiteral3; + QuantityLiteralPath ucumQuantityLiteral1; + QuantityLiteralPath ucumQuantityLiteral2; + QuantityLiteralPath ucumQuantityLiteral3; + QuantityLiteralPath calendarDurationLiteral1; + QuantityLiteralPath calendarDurationLiteral2; + QuantityLiteralPath calendarDurationLiteral3; @BeforeEach void setUp() { @@ -86,28 +86,28 @@ void setUp() { quantity4.setCode("258682000"); final Quantity quantity5 = new Quantity(); - quantity1.setValue(650); - quantity1.setUnit("mg"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("mg"); + quantity5.setValue(650); + quantity5.setUnit("mg"); + quantity5.setSystem(TestHelpers.UCUM_URL); + quantity5.setCode("mg"); final Quantity quantity6 = new Quantity(); - quantity1.setValue(30); - quantity1.setUnit("d"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("d"); + quantity6.setValue(30); + quantity6.setUnit("d"); + quantity6.setSystem(TestHelpers.UCUM_URL); + quantity6.setCode("d"); final Quantity quantity7 = new Quantity(); - quantity1.setValue(60); - quantity1.setUnit("s"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("s"); + quantity7.setValue(60); + quantity7.setUnit("s"); + quantity7.setSystem(TestHelpers.UCUM_URL); + quantity7.setCode("s"); final Quantity quantity8 = new Quantity(); - quantity1.setValue(1000); - quantity1.setUnit("ms"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("ms"); + quantity8.setValue(1000); + quantity8.setUnit("ms"); + quantity8.setSystem(TestHelpers.UCUM_URL); + quantity8.setCode("ms"); final Dataset leftDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) @@ -149,12 +149,13 @@ void setUp() { .idAndValueColumns() .build(); - ucumQuantityLiteral1 = UcumQuantityLiteralPath.fromString("500 'mg'", left, ucumService); - ucumQuantityLiteral2 = UcumQuantityLiteralPath.fromString("0.5 'g'", left, ucumService); - ucumQuantityLiteral3 = UcumQuantityLiteralPath.fromString("1.8 'm'", left, ucumService); - calendarDurationLiteral1 = CalendarDurationLiteralPath.fromString("30 days", left); - calendarDurationLiteral2 = CalendarDurationLiteralPath.fromString("60 seconds", left); - calendarDurationLiteral3 = CalendarDurationLiteralPath.fromString("1000 milliseconds", left); + ucumQuantityLiteral1 = QuantityLiteralPath.fromUcumString("500 'mg'", left, ucumService); + ucumQuantityLiteral2 = QuantityLiteralPath.fromUcumString("0.5 'g'", left, ucumService); + ucumQuantityLiteral3 = QuantityLiteralPath.fromUcumString("1.8 'm'", left, ucumService); + calendarDurationLiteral1 = QuantityLiteralPath.fromCalendarDurationString("30 days", left); + calendarDurationLiteral2 = QuantityLiteralPath.fromCalendarDurationString("60 seconds", left); + calendarDurationLiteral3 = QuantityLiteralPath.fromCalendarDurationString("1000 milliseconds", + left); parserContext = new ParserContextBuilder(spark, fhirContext) .groupingColumns(Collections.singletonList(left.getIdColumn())) @@ -251,6 +252,8 @@ void ucumLiteralEquals() { result = equalityOperator.invoke(input); final Dataset allTrue = new DatasetBuilder(spark) + .withIdColumn() + .withColumn(DataTypes.BooleanType) .withIdsAndValue(true, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", "patient-7", "patient-8", "patient-9")) @@ -310,6 +313,8 @@ void ucumLiteralNotEquals() { result = equalityOperator.invoke(input); final Dataset allFalse = new DatasetBuilder(spark) + .withIdColumn() + .withColumn(DataTypes.BooleanType) .withIdsAndValue(false, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", "patient-7", "patient-8", "patient-9")) @@ -330,7 +335,7 @@ void calendarLiteralEquals() { RowFactory.create("patient-4", null), // 650 mg = 30 days RowFactory.create("patient-5", null), // {} = 30 days RowFactory.create("patient-6", null), // 500 mg = 30 days - RowFactory.create("patient-7", true), // 30 d = 30 days + RowFactory.create("patient-7", null), // 30 d = 30 days RowFactory.create("patient-8", null), // 60 s = 30 days RowFactory.create("patient-9", null) // 1000 ms = 30 days ); @@ -339,36 +344,38 @@ void calendarLiteralEquals() { result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", null), // 500 mg = 60 s - RowFactory.create("patient-2", null), // 500 mg = 60 s - RowFactory.create("patient-3", null), // 500 mg = 60 s - RowFactory.create("patient-4", null), // 650 mg = 60 s - RowFactory.create("patient-5", null), // {} = 60 s - RowFactory.create("patient-6", null), // 500 mg = 60 s - RowFactory.create("patient-7", null), // 30 d = 60 s - RowFactory.create("patient-8", true), // 60 s = 60 s - RowFactory.create("patient-9", false) // 1000 ms = 60 s + RowFactory.create("patient-1", null), // 500 mg = 60 seconds + RowFactory.create("patient-2", null), // 500 mg = 60 seconds + RowFactory.create("patient-3", null), // 500 mg = 60 seconds + RowFactory.create("patient-4", null), // 650 mg = 60 seconds + RowFactory.create("patient-5", null), // {} = 60 seconds + RowFactory.create("patient-6", null), // 500 mg = 60 seconds + RowFactory.create("patient-7", false), // 30 d = 60 seconds + RowFactory.create("patient-8", true), // 60 s = 60 seconds + RowFactory.create("patient-9", false) // 1000 ms = 60 seconds ); input = new OperatorInput(parserContext, left, calendarDurationLiteral3); result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", null), // 500 mg = 1000 ms - RowFactory.create("patient-2", null), // 500 mg = 1000 ms - RowFactory.create("patient-3", null), // 500 mg = 1000 ms - RowFactory.create("patient-4", null), // 650 mg = 1000 ms - RowFactory.create("patient-5", null), // {} = 1000 ms - RowFactory.create("patient-6", null), // 500 mg = 1000 ms - RowFactory.create("patient-7", null), // 30 d = 1000 ms - RowFactory.create("patient-8", false), // 60 s = 1000 ms - RowFactory.create("patient-9", true) // 1000 ms = 1000 ms + RowFactory.create("patient-1", null), // 500 mg = 1000 milliseconds + RowFactory.create("patient-2", null), // 500 mg = 1000 milliseconds + RowFactory.create("patient-3", null), // 500 mg = 1000 milliseconds + RowFactory.create("patient-4", null), // 650 mg = 1000 milliseconds + RowFactory.create("patient-5", null), // {} = 1000 milliseconds + RowFactory.create("patient-6", null), // 500 mg = 1000 milliseconds + RowFactory.create("patient-7", false), // 30 d = 1000 milliseconds + RowFactory.create("patient-8", false), // 60 s = 1000 milliseconds + RowFactory.create("patient-9", true) // 1000 ms = 1000 milliseconds ); - input = new OperatorInput(parserContext, calendarDurationLiteral1, calendarDurationLiteral1); + input = new OperatorInput(parserContext, calendarDurationLiteral2, calendarDurationLiteral2); result = equalityOperator.invoke(input); final Dataset allTrue = new DatasetBuilder(spark) + .withIdColumn() + .withColumn(DataTypes.BooleanType) .withIdsAndValue(true, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", "patient-7", "patient-8", "patient-9")) @@ -389,7 +396,7 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-4", null), // 650 mg != 30 days RowFactory.create("patient-5", null), // {} != 30 days RowFactory.create("patient-6", null), // 500 mg != 30 days - RowFactory.create("patient-7", false), // 30 d != 30 days + RowFactory.create("patient-7", null), // 30 d != 30 days RowFactory.create("patient-8", null), // 60 s != 30 days RowFactory.create("patient-9", null) // 1000 ms != 30 days ); @@ -404,7 +411,7 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-4", null), // 650 mg != 60 s RowFactory.create("patient-5", null), // {} != 60 s RowFactory.create("patient-6", null), // 500 mg != 60 s - RowFactory.create("patient-7", null), // 30 d != 60 s + RowFactory.create("patient-7", true), // 30 d != 60 s RowFactory.create("patient-8", false), // 60 s != 60 s RowFactory.create("patient-9", true) // 1000 ms != 60 s ); @@ -419,15 +426,17 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-4", null), // 650 mg != 1000 ms RowFactory.create("patient-5", null), // {} != 1000 ms RowFactory.create("patient-6", null), // 500 mg != 1000 ms - RowFactory.create("patient-7", null), // 30 d != 1000 ms + RowFactory.create("patient-7", true), // 30 d != 1000 ms RowFactory.create("patient-8", true), // 60 s != 1000 ms RowFactory.create("patient-9", false) // 1000 ms != 1000 ms ); - input = new OperatorInput(parserContext, calendarDurationLiteral1, calendarDurationLiteral1); + input = new OperatorInput(parserContext, calendarDurationLiteral2, calendarDurationLiteral2); result = equalityOperator.invoke(input); final Dataset allFalse = new DatasetBuilder(spark) + .withIdColumn() + .withColumn(DataTypes.BooleanType) .withIdsAndValue(false, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", "patient-7", "patient-8", "patient-9")) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index f9149267c4..ed0ff9cf41 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -13,7 +13,7 @@ import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; import au.csiro.pathling.terminology.TerminologyService; -import au.csiro.pathling.terminology.ucum.QuantityEquality; +import au.csiro.pathling.terminology.ucum.ComparableQuantity; import au.csiro.pathling.terminology.ucum.Ucum; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; @@ -42,8 +42,8 @@ class UnitTestDependencies { static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, - @Nonnull final QuantityEquality quantityEquality) { - return Spark.build(configuration, environment, sparkListener, quantityEquality); + @Nonnull final ComparableQuantity comparableQuantity) { + return Spark.build(configuration, environment, sparkListener, comparableQuantity); } @Bean diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 6dbf20c3eb..79467d5bc2 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -31,6 +31,7 @@ import org.hl7.fhir.r4.model.CodeableConcept; import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Quantity; +import org.hl7.fhir.r4.model.Quantity.QuantityComparator; import scala.collection.JavaConverters; import scala.collection.mutable.Buffer; @@ -40,6 +41,8 @@ @SuppressWarnings("WeakerAccess") public abstract class SparkHelpers { + public static final int DECIMAL_SCALE = 2; + @Nonnull public static IdAndValueColumns getIdAndValueColumns(@Nonnull final Dataset dataset) { return getIdAndValueColumns(dataset, false); @@ -107,10 +110,17 @@ public static StructType quantityStructType() { final StructField id = new StructField("id", DataTypes.StringType, true, metadata); final StructField value = new StructField("value", DataTypes.createDecimalType( DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); + final StructField valueScale = new StructField("value_scale", DataTypes.IntegerType, true, + metadata); + final StructField comparator = new StructField("comparator", DataTypes.StringType, true, + metadata); final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); final StructField system = new StructField("system", DataTypes.StringType, true, metadata); final StructField code = new StructField("code", DataTypes.StringType, true, metadata); - return new StructType(new StructField[]{id, value, unit, system, code}); + final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, + metadata); + return new StructType( + new StructField[]{id, value, valueScale, comparator, unit, system, code, fid}); } @Nonnull @@ -154,9 +164,11 @@ public static Row rowFromCodeableConcept(@Nonnull final CodeableConcept codeable @Nonnull public static Row rowFromQuantity(@Nonnull final Quantity quantity) { + final String comparator = Optional.ofNullable(quantity.getComparator()) + .map(QuantityComparator::toCode).orElse(null); return new GenericRowWithSchema( - new Object[]{quantity.getId(), quantity.getValue(), quantity.getUnit(), - quantity.getSystem(), quantity.getCode()}, + new Object[]{quantity.getId(), quantity.getValue(), DECIMAL_SCALE, comparator, + quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */}, quantityStructType()); } diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index a0a55e7f77..94d19c0fc3 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -58,7 +58,8 @@ All comparison operators return a [Boolean](./data-types.html#boolean) value. * Not all Quantity, Date and DateTime values are comparable, it depends upon the comparability of the units within the Quantity values. See the [FHIRPath specification](https://hl7.org/fhirpath/#comparison) for details on -how Quantity values are compared. +how Quantity values are compared. Quantities with a `comparator` are treated as +not comparable by this implementation. See also: [Comparison](https://hl7.org/fhirpath/#comparison) @@ -78,7 +79,8 @@ an empty collection. Not all Quantity, Date and DateTime values can be compared for equality, it depends upon the comparability of the units within the Quantity values. See the [FHIRPath specification](https://hl7.org/fhirpath/#quantity-equality) for -details on how equality works with Quantity values. +details on how equality works with Quantity values. Quantities with a +`comparator` are treated as not comparable by this implementation. See also: [Equality](https://hl7.org/fhirpath/#equality) From 9016748944e93bd0dbedf1b3681835ea9deefc98 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 7 Apr 2022 14:49:27 +1000 Subject: [PATCH 07/83] Get ComparisonOperatorQuantityTest passing --- .../terminology/ucum/ComparableQuantity.java | 1 + .../ComparisonOperatorQuantityTest.java | 148 +++++++++--------- 2 files changed, 75 insertions(+), 74 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index 6638fe678e..8e1d159856 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -72,6 +72,7 @@ public Row call(@Nullable final Row row) throws Exception { resolvedCode = input.getCode(); } else if (input.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI) && CALENDAR_DURATION_TO_UCUM.containsKey(input.getCode())) { + // If it is a comparable calendar duration, convert it to UCUM. resolvedCode = CALENDAR_DURATION_TO_UCUM.get(input.getCode()); } else { // If the Quantity is not UCUM or a comparable calendar duration, it is not comparable. diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java index 3244fb26b3..5450d8ea54 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorQuantityTest.java @@ -84,47 +84,47 @@ void setUp() { quantity4.setCode("258682000"); final Quantity quantity5 = new Quantity(); - quantity1.setValue(650); - quantity1.setUnit("mg"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("mg"); + quantity5.setValue(650); + quantity5.setUnit("mg"); + quantity5.setSystem(TestHelpers.UCUM_URL); + quantity5.setCode("mg"); final Quantity quantity6 = new Quantity(); - quantity1.setValue(30); - quantity1.setUnit("d"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("d"); + quantity6.setValue(30); + quantity6.setUnit("d"); + quantity6.setSystem(TestHelpers.UCUM_URL); + quantity6.setCode("d"); final Quantity quantity7 = new Quantity(); - quantity1.setValue(60); - quantity1.setUnit("s"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("s"); + quantity7.setValue(60); + quantity7.setUnit("s"); + quantity7.setSystem(TestHelpers.UCUM_URL); + quantity7.setCode("s"); final Quantity quantity8 = new Quantity(); - quantity1.setValue(1000); - quantity1.setUnit("ms"); - quantity1.setSystem(TestHelpers.UCUM_URL); - quantity1.setCode("ms"); + quantity8.setValue(1000); + quantity8.setUnit("ms"); + quantity8.setSystem(TestHelpers.UCUM_URL); + quantity8.setCode("ms"); final Quantity quantity9 = new Quantity(); - quantity2.setValue(0.2); - quantity2.setUnit("g"); - quantity2.setSystem(TestHelpers.UCUM_URL); - quantity2.setCode("g"); + quantity9.setValue(0.2); + quantity9.setUnit("g"); + quantity9.setSystem(TestHelpers.UCUM_URL); + quantity9.setCode("g"); final Dataset leftDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) .withStructTypeColumns(quantityStructType()) - .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-2", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-3", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-4", rowFromQuantity(quantity5)) // 650 mg - .withRow("patient-5", null) - .withRow("patient-6", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d - .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s - .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-01", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-02", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-03", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-04", rowFromQuantity(quantity5)) // 650 mg + .withRow("patient-05", null) + .withRow("patient-06", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-07", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-08", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-09", rowFromQuantity(quantity8)) // 1000 ms .withRow("patient-10", rowFromQuantity(quantity9)) // 0.2 g .buildWithStructValue(); left = new ElementPathBuilder(spark) @@ -137,15 +137,15 @@ void setUp() { final Dataset rightDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) .withStructTypeColumns(quantityStructType()) - .withRow("patient-1", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-2", rowFromQuantity(quantity2)) // 0.5 g - .withRow("patient-3", rowFromQuantity(quantity3)) // 1.8 m - .withRow("patient-4", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-5", rowFromQuantity(quantity1)) // 500 mg - .withRow("patient-6", null) - .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d - .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s - .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-01", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-02", rowFromQuantity(quantity2)) // 0.5 g + .withRow("patient-03", rowFromQuantity(quantity3)) // 1.8 m + .withRow("patient-04", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-05", rowFromQuantity(quantity1)) // 500 mg + .withRow("patient-06", null) + .withRow("patient-07", rowFromQuantity(quantity6)) // 30 d + .withRow("patient-08", rowFromQuantity(quantity7)) // 60 s + .withRow("patient-09", rowFromQuantity(quantity8)) // 1000 ms .withRow("patient-10", rowFromQuantity(quantity1)) // 500 mg .buildWithStructValue(); right = new ElementPathBuilder(spark) @@ -175,15 +175,15 @@ void lessThan() { final FhirPath result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", false), // 500 mg < 500 mg - RowFactory.create("patient-2", false), // 500 mg < 0.5 g - RowFactory.create("patient-3", null), // 500 mg < 1.8 m - RowFactory.create("patient-4", false), // 650 mg < 500 mg - RowFactory.create("patient-5", null), // {} < 500 mg - RowFactory.create("patient-6", null), // 500 mg < {} - RowFactory.create("patient-7", false), // 30 d < 30 d - RowFactory.create("patient-8", false), // 60 s < 60 s - RowFactory.create("patient-9", false), // 1000 ms < 1000 ms + RowFactory.create("patient-01", false), // 500 mg < 500 mg + RowFactory.create("patient-02", false), // 500 mg < 0.5 g + RowFactory.create("patient-03", null), // 500 mg < 1.8 m + RowFactory.create("patient-04", false), // 650 mg < 500 mg + RowFactory.create("patient-05", null), // {} < 500 mg + RowFactory.create("patient-06", null), // 500 mg < {} + RowFactory.create("patient-07", false), // 30 d < 30 d + RowFactory.create("patient-08", false), // 60 s < 60 s + RowFactory.create("patient-09", false), // 1000 ms < 1000 ms RowFactory.create("patient-10", true) // 0.2 g < 500 mg ); } @@ -195,15 +195,15 @@ void lessThanOrEqualTo() { final FhirPath result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", true), // 500 mg <= 500 mg - RowFactory.create("patient-2", true), // 500 mg <= 0.5 g - RowFactory.create("patient-3", null), // 500 mg <= 1.8 m - RowFactory.create("patient-4", false), // 650 mg <= 500 mg - RowFactory.create("patient-5", null), // {} <= 500 mg - RowFactory.create("patient-6", null), // 500 mg <= {} - RowFactory.create("patient-7", true), // 30 d <= 30 d - RowFactory.create("patient-8", true), // 60 s <= 60 s - RowFactory.create("patient-9", true), // 1000 ms <= 1000 ms + RowFactory.create("patient-01", true), // 500 mg <= 500 mg + RowFactory.create("patient-02", true), // 500 mg <= 0.5 g + RowFactory.create("patient-03", null), // 500 mg <= 1.8 m + RowFactory.create("patient-04", false), // 650 mg <= 500 mg + RowFactory.create("patient-05", null), // {} <= 500 mg + RowFactory.create("patient-06", null), // 500 mg <= {} + RowFactory.create("patient-07", true), // 30 d <= 30 d + RowFactory.create("patient-08", true), // 60 s <= 60 s + RowFactory.create("patient-09", true), // 1000 ms <= 1000 ms RowFactory.create("patient-10", true) // 0.2 g <= 500 mg ); } @@ -215,15 +215,15 @@ void greaterThanOrEqualTo() { final FhirPath result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", true), // 500 mg >= 500 mg - RowFactory.create("patient-2", true), // 500 mg >= 0.5 g - RowFactory.create("patient-3", null), // 500 mg >= 1.8 m - RowFactory.create("patient-4", true), // 650 mg >= 500 mg - RowFactory.create("patient-5", null), // {} >= 500 mg - RowFactory.create("patient-6", null), // 500 mg >= {} - RowFactory.create("patient-7", true), // 30 d >= 30 d - RowFactory.create("patient-8", true), // 60 s >= 60 s - RowFactory.create("patient-9", true), // 1000 ms >= 1000 ms + RowFactory.create("patient-01", true), // 500 mg >= 500 mg + RowFactory.create("patient-02", true), // 500 mg >= 0.5 g + RowFactory.create("patient-03", null), // 500 mg >= 1.8 m + RowFactory.create("patient-04", true), // 650 mg >= 500 mg + RowFactory.create("patient-05", null), // {} >= 500 mg + RowFactory.create("patient-06", null), // 500 mg >= {} + RowFactory.create("patient-07", true), // 30 d >= 30 d + RowFactory.create("patient-08", true), // 60 s >= 60 s + RowFactory.create("patient-09", true), // 1000 ms >= 1000 ms RowFactory.create("patient-10", false) // 0.2 g >= 500 mg ); } @@ -235,15 +235,15 @@ void greaterThan() { final FhirPath result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", false), // 500 mg > 500 mg - RowFactory.create("patient-2", false), // 500 mg > 0.5 g - RowFactory.create("patient-3", null), // 500 mg > 1.8 m - RowFactory.create("patient-4", true), // 650 mg > 500 mg - RowFactory.create("patient-5", null), // {} > 500 mg - RowFactory.create("patient-6", null), // 500 mg > {} - RowFactory.create("patient-7", false), // 30 d > 30 d - RowFactory.create("patient-8", false), // 60 s > 60 s - RowFactory.create("patient-9", false), // 1000 ms > 1000 ms + RowFactory.create("patient-01", false), // 500 mg > 500 mg + RowFactory.create("patient-02", false), // 500 mg > 0.5 g + RowFactory.create("patient-03", null), // 500 mg > 1.8 m + RowFactory.create("patient-04", true), // 650 mg > 500 mg + RowFactory.create("patient-05", null), // {} > 500 mg + RowFactory.create("patient-06", null), // 500 mg > {} + RowFactory.create("patient-07", false), // 30 d > 30 d + RowFactory.create("patient-08", false), // 60 s > 60 s + RowFactory.create("patient-09", false), // 1000 ms > 1000 ms RowFactory.create("patient-10", false) // 0.2 g > 500 mg ); } From eae8b3f7072c9bf9a765471619178d1d495041cc Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 7 Apr 2022 16:45:46 +1000 Subject: [PATCH 08/83] Get MathOperatorQuantityTest passing --- .../au/csiro/pathling/fhirpath/Numeric.java | 28 +++++++ .../fhirpath/element/DecimalPath.java | 27 ++++--- .../fhirpath/element/IntegerPath.java | 26 ++++--- .../fhirpath/element/QuantityPath.java | 74 ++++++++++++++++++- .../fhirpath/literal/DecimalLiteralPath.java | 25 ++++++- .../fhirpath/literal/IntegerLiteralPath.java | 28 ++++++- .../fhirpath/literal/QuantityLiteralPath.java | 32 +++++++- .../terminology/ucum/ComparableQuantity.java | 3 +- .../operator/MathOperatorQuantityTest.java | 5 +- 9 files changed, 216 insertions(+), 32 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java index f94c5e58bb..9692081687 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java @@ -13,6 +13,7 @@ import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; /** * Describes a path that represents a numeric value, and can be the subject of math operations. @@ -48,6 +49,33 @@ Function getMathOperation(@Nonnull MathOperation operat @Nonnull Column getValueColumn(); + /** + * @return a {@link Column} that provides a value that can me used in math operations + */ + @Nonnull + Column getNumericValueColumn(); + + /** + * Provides a {@link Column} that provides additional context that informs the way that math + * operations are carried out. This is used for Quantity math, so that the operation function has + * access to the canonicalized units. + * + * @return a {@link Column} that provides additional context for math operations + */ + @Nonnull + Column getNumericContextColumn(); + + /** + * The FHIR data type of the element being represented by this expression. + *

+ * Note that there can be multiple valid FHIR types for a given FHIRPath type, e.g. {@code uri} + * and {@code code} both map to the {@code String} FHIRPath type. + * + * @see Using FHIR types in expressions + */ + @Nonnull + FHIRDefinedType getFhirType(); + /** * Represents a type of math operator. */ diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java index ea76e9d89d..21501e75bb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java @@ -15,7 +15,6 @@ import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.literal.DecimalLiteralPath; -import au.csiro.pathling.fhirpath.literal.IntegerLiteralPath; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Optional; @@ -34,7 +33,6 @@ * * @author John Grimes */ -@SuppressWarnings("NullableProblems") public class DecimalPath extends ElementPath implements Materializable, Comparable, Numeric { @@ -109,7 +107,19 @@ public boolean isComparableTo(@Nonnull final Class type) { @Override public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { - return buildMathOperation(this, operation, expression, dataset, getFhirType()); + return buildMathOperation(this, operation, expression, dataset); + } + + @Nonnull + @Override + public Column getNumericValueColumn() { + return getValueColumn(); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return getNumericValueColumn(); } /** @@ -119,21 +129,16 @@ public Function getMathOperation(@Nonnull final MathOpe * @param operation The type of {@link au.csiro.pathling.fhirpath.Numeric.MathOperation} * @param expression The FHIRPath expression to use in the result * @param dataset The {@link Dataset} to use in the result - * @param fhirType The {@link FHIRDefinedType} to use in the result * @return A {@link Function} that takes a {@link Numeric} as a parameter, and returns a {@link * NonLiteralPath} */ @Nonnull public static Function buildMathOperation(@Nonnull final Numeric source, @Nonnull final MathOperation operation, @Nonnull final String expression, - @Nonnull final Dataset dataset, @Nonnull final FHIRDefinedType fhirType) { + @Nonnull final Dataset dataset) { return target -> { - final Column targetValueColumn = - target instanceof IntegerPath || target instanceof IntegerLiteralPath - ? target.getValueColumn().cast(DataTypes.LongType) - : target.getValueColumn(); Column valueColumn = operation.getSparkFunction() - .apply(source.getValueColumn(), targetValueColumn); + .apply(source.getNumericValueColumn(), target.getNumericValueColumn()); final Column idColumn = source.getIdColumn(); final Optional eidColumn = findEidColumn(source, target); final Optional thisColumn = findThisColumn(source, target); @@ -146,7 +151,7 @@ public static Function buildMathOperation(@Nonnull fina valueColumn = valueColumn.cast(getDecimalType()); return ElementPath .build(expression, dataset, idColumn, eidColumn, valueColumn, true, Optional.empty(), - thisColumn, fhirType); + thisColumn, source.getFhirType()); case MODULUS: valueColumn = valueColumn.cast(DataTypes.LongType); return ElementPath diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java index 006a579dbb..b71adaa974 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java @@ -118,7 +118,19 @@ public boolean isComparableTo(@Nonnull final Class type) { @Override public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { - return buildMathOperation(this, operation, expression, dataset, getFhirType()); + return buildMathOperation(this, operation, expression, dataset); + } + + @Nonnull + @Override + public Column getNumericValueColumn() { + return getValueColumn().cast(DataTypes.LongType); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return getNumericValueColumn(); } /** @@ -128,21 +140,17 @@ public Function getMathOperation(@Nonnull final MathOpe * @param operation The type of {@link au.csiro.pathling.fhirpath.Numeric.MathOperation} * @param expression The FHIRPath expression to use in the result * @param dataset The {@link Dataset} to use in the result - * @param fhirType The {@link FHIRDefinedType} to use in the result * @return A {@link Function} that takes a {@link Numeric} as a parameter, and returns a {@link * NonLiteralPath} */ @Nonnull public static Function buildMathOperation(@Nonnull final Numeric source, @Nonnull final MathOperation operation, @Nonnull final String expression, - @Nonnull final Dataset dataset, @Nonnull final FHIRDefinedType fhirType) { + @Nonnull final Dataset dataset) { return target -> { - final Column targetValueColumn = - target instanceof IntegerPath || target instanceof IntegerLiteralPath - ? target.getValueColumn().cast(DataTypes.LongType) - : target.getValueColumn(); + final Column targetValueColumn = target.getNumericValueColumn(); Column valueColumn = operation.getSparkFunction() - .apply(source.getValueColumn().cast(DataTypes.LongType), targetValueColumn); + .apply(source.getNumericValueColumn(), targetValueColumn); final Column idColumn = source.getIdColumn(); final Optional eidColumn = findEidColumn(source, target); final Optional thisColumn = findThisColumn(source, target); @@ -157,7 +165,7 @@ public static Function buildMathOperation(@Nonnull fina } return ElementPath .build(expression, dataset, idColumn, eidColumn, valueColumn, true, Optional.empty(), - thisColumn, fhirType); + thisColumn, source.getFhirType()); case DIVISION: final Column numerator = source.getValueColumn().cast(DecimalPath.getDecimalType()); valueColumn = operation.getSparkFunction().apply(numerator, targetValueColumn); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 0790782e33..02a1ec607b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -7,10 +7,13 @@ package au.csiro.pathling.fhirpath.element; import static org.apache.spark.sql.functions.callUDF; +import static org.apache.spark.sql.functions.struct; import static org.apache.spark.sql.functions.when; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; @@ -29,7 +32,7 @@ * * @author John Grimes */ -public class QuantityPath extends ElementPath implements Comparable { +public class QuantityPath extends ElementPath implements Comparable, Numeric { public static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(QuantityPath.class, QuantityLiteralPath.class, NullLiteralPath.class); @@ -71,4 +74,73 @@ public boolean isComparableTo(@Nonnull final Class type) { return COMPARABLE_TYPES.contains(type); } + @Nonnull + @Override + public Column getNumericValueColumn() { + final Column comparable = buildComparableValueColumn(this); + return comparable.getField("value"); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return buildComparableValueColumn(this); + } + + @Nonnull + public static Column buildComparableValueColumn(@Nonnull final Numeric source) { + return callUDF(ComparableQuantity.FUNCTION_NAME, source.getValueColumn()); + } + + @Nonnull + @Override + public Function getMathOperation(@Nonnull final MathOperation operation, + @Nonnull final String expression, @Nonnull final Dataset dataset) { + return buildMathOperation(this, operation, expression, dataset, getFhirType()); + } + + @Nonnull + public static Function buildMathOperation(@Nonnull final Numeric source, + @Nonnull final MathOperation operation, @Nonnull final String expression, + @Nonnull final Dataset dataset, @Nonnull final FHIRDefinedType fhirType) { + return target -> { + final Column sourceQuantity = source.getValueColumn(); + final Column sourceComparable = source.getNumericValueColumn(); + final Column resultColumn = operation.getSparkFunction() + .apply(sourceComparable, target.getNumericValueColumn()); + final Column resultStruct = struct( + sourceQuantity.getField("id").as("id"), + resultColumn.as("value"), + sourceQuantity.getField("value_scale").as("value_scale"), + sourceQuantity.getField("comparator").as("comparator"), + sourceQuantity.getField("unit").as("unit"), + sourceQuantity.getField("system").as("system"), + sourceQuantity.getField("code").as("code"), + sourceQuantity.getField("_fid").as("_fid") + ); + final Column sourceCanonicalizedCode = source.getNumericContextColumn().getField("code"); + final Column targetCanonicalizedCode = target.getNumericContextColumn().getField("code"); + final Column resultQuantityColumn = + when(sourceCanonicalizedCode.equalTo(targetCanonicalizedCode), resultStruct) + .otherwise(null); + + final Column idColumn = source.getIdColumn(); + final Optional eidColumn = findEidColumn(source, target); + final Optional thisColumn = findThisColumn(source, target); + + switch (operation) { + case ADDITION: + case SUBTRACTION: + case MULTIPLICATION: + case DIVISION: + return ElementPath + .build(expression, dataset, idColumn, eidColumn, resultQuantityColumn, true, + Optional.empty(), + thisColumn, fhirType); + default: + throw new AssertionError("Unsupported math operation encountered: " + operation); + } + }; + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java index 5961e6b6e8..a2ad52184a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java @@ -10,7 +10,10 @@ import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.*; +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.element.DecimalPath; import au.csiro.pathling.fhirpath.element.IntegerPath; import java.math.BigDecimal; @@ -101,7 +104,25 @@ public boolean isComparableTo(@Nonnull final Class type) { public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { return DecimalPath - .buildMathOperation(this, operation, expression, dataset, FHIRDefinedType.DECIMAL); + .buildMathOperation(this, operation, expression, dataset); + } + + @Nonnull + @Override + public Column getNumericValueColumn() { + return getValueColumn(); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return getNumericValueColumn(); + } + + @Nonnull + @Override + public FHIRDefinedType getFhirType() { + return FHIRDefinedType.DECIMAL; } @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java index 62e912319d..bdf8c5f905 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java @@ -9,7 +9,10 @@ import static au.csiro.pathling.utilities.Preconditions.check; import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.*; +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.element.IntegerPath; import java.util.Optional; import java.util.function.Function; @@ -17,6 +20,7 @@ import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.apache.spark.sql.types.DataTypes; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.IntegerType; import org.hl7.fhir.r4.model.PrimitiveType; @@ -87,7 +91,25 @@ public boolean isComparableTo(@Nonnull final Class type) { public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { return IntegerPath - .buildMathOperation(this, operation, expression, dataset, FHIRDefinedType.INTEGER); + .buildMathOperation(this, operation, expression, dataset); + } + + @Nonnull + @Override + public Column getNumericValueColumn() { + return getValueColumn().cast(DataTypes.LongType); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return getNumericValueColumn(); + } + + @Nonnull + @Override + public FHIRDefinedType getFhirType() { + return FHIRDefinedType.INTEGER; } @Nonnull @@ -100,5 +122,5 @@ public Optional getValueFromRow(@Nonnull final Row row, final int public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof IntegerPath; } - + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 11aa97a79e..3a7b92ac52 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -13,6 +13,8 @@ import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.terminology.ucum.Ucum; import java.math.BigDecimal; @@ -26,6 +28,7 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; import org.hl7.fhir.r4.model.Type; @@ -36,7 +39,7 @@ * @author John Grimes */ @Getter -public class QuantityLiteralPath extends LiteralPath implements Comparable { +public class QuantityLiteralPath extends LiteralPath implements Comparable, Numeric { public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; @@ -170,4 +173,31 @@ public boolean isComparableTo(@Nonnull final Class type) { return QuantityPath.COMPARABLE_TYPES.contains(type); } + @Nonnull + @Override + public Column getNumericValueColumn() { + final Column comparable = QuantityPath.buildComparableValueColumn(this); + return comparable.getField("value"); + } + + @Nonnull + @Override + public Column getNumericContextColumn() { + return QuantityPath.buildComparableValueColumn(this); + } + + @Nonnull + @Override + public FHIRDefinedType getFhirType() { + return FHIRDefinedType.QUANTITY; + } + + @Nonnull + @Override + public Function getMathOperation(@Nonnull final MathOperation operation, + @Nonnull final String expression, @Nonnull final Dataset dataset) { + return QuantityPath.buildMathOperation(this, operation, expression, dataset, + FHIRDefinedType.QUANTITY); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index 8e1d159856..5ce5fef3bd 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -87,8 +87,7 @@ public Row call(@Nullable final Row row) throws Exception { // Create a new Quantity object with the canonicalized result. final Quantity result = new Quantity(); result.setValue(new BigDecimal(canonical.getValue().asDecimal())); - result.setComparator(input.getComparator()); - result.setUnit(input.getUnit()); + result.setUnit(canonical.getCode()); result.setSystem(Ucum.SYSTEM_URI); result.setCode(canonical.getCode()); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java index 6b35b67959..b4f776627a 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java @@ -49,7 +49,7 @@ public class MathOperatorQuantityTest { @Autowired FhirContext fhirContext; - static final List OPERATORS = List.of("+", "-", "*", "/", "mod"); + static final List OPERATORS = List.of("+", "-", "*", "/"); static final String ID_ALIAS = "_abc123"; @Value @@ -109,7 +109,6 @@ Dataset expectedResult(@Nonnull final String operator) { result = rowForUcumQuantity(225000.0, "m"); break; case "/": - case "mod": result = rowForUcumQuantity(0.1, "m"); break; default: @@ -128,7 +127,7 @@ Dataset expectedResult(@Nonnull final String operator) { } @Nonnull - FhirPath buildQuantityExpression(@Nonnull final boolean leftOperand) { + FhirPath buildQuantityExpression(final boolean leftOperand) { final Quantity nonUcumQuantity = new Quantity(); nonUcumQuantity.setValue(15); nonUcumQuantity.setUnit("mSv"); From 56657a639e1c3ec8e99e550480b20ac7426ae2d7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 9 Apr 2022 12:48:24 +1000 Subject: [PATCH 09/83] Get DateArithmeticTest passing --- .../pathling/aggregate/AggregateExecutor.java | 4 +- .../au/csiro/pathling/fhirpath/Temporal.java | 57 ++ .../pathling/fhirpath/element/DatePath.java | 19 +- .../fhirpath/element/DateTimePath.java | 61 +- .../function/memberof/MemberOfFunction.java | 4 +- .../function/subsumes/SubsumesFunction.java | 4 +- .../function/translate/TranslateFunction.java | 4 +- .../fhirpath/literal/DateLiteralPath.java | 17 +- .../fhirpath/literal/DateTimeLiteralPath.java | 38 +- .../operator/DateArithmeticOperator.java | 67 +++ .../fhirpath/operator/MathOperator.java | 8 + .../java/au/csiro/pathling/spark/Spark.java | 27 +- ...lingFunctions.java => SqlExpressions.java} | 2 +- ...{SqlExtensions.java => SqlOperations.java} | 2 +- .../pathling/sql/dates/AddDurationToDate.java | 36 ++ .../sql/dates/AddDurationToDateTime.java | 36 ++ .../sql/dates/DateArithmeticFunction.java | 34 ++ .../sql/dates/DateTimeArithmeticFunction.java | 31 + .../sql/dates/SubtractDurationFromDate.java | 36 ++ .../dates/SubtractDurationFromDateTime.java | 33 ++ .../sql/dates/TemporalArithmeticFunction.java | 109 ++++ .../csiro/pathling/sql/udf/SqlFunction1.java | 18 + .../csiro/pathling/sql/udf/SqlFunction2.java | 18 + .../pathling/terminology/CodingToLiteral.java | 20 +- .../terminology/ucum/ComparableQuantity.java | 17 +- ...thlingStrategy.scala => SqlStrategy.scala} | 10 +- .../fhirpath/operator/DateArithmeticTest.java | 545 ++++++------------ .../sql/PruneSyntheticFieldsTest.java | 4 +- ...nsionsTest.java => SqlOperationsTest.java} | 6 +- .../pathling/test/UnitTestDependencies.java | 17 +- 30 files changed, 859 insertions(+), 425 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java rename fhir-server/src/main/java/au/csiro/pathling/sql/{PathlingFunctions.java => SqlExpressions.java} (95%) rename fhir-server/src/main/java/au/csiro/pathling/sql/{SqlExtensions.java => SqlOperations.java} (98%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java rename fhir-server/src/main/scala/au/csiro/pathling/sql/{PathlingStrategy.scala => SqlStrategy.scala} (71%) rename fhir-server/src/test/java/au/csiro/pathling/sql/{SqlExtensionsTest.java => SqlOperationsTest.java} (96%) diff --git a/fhir-server/src/main/java/au/csiro/pathling/aggregate/AggregateExecutor.java b/fhir-server/src/main/java/au/csiro/pathling/aggregate/AggregateExecutor.java index 01c1bd54fc..b69b956bc9 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/aggregate/AggregateExecutor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/aggregate/AggregateExecutor.java @@ -19,7 +19,7 @@ import au.csiro.pathling.fhirpath.parser.Parser; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.io.Database; -import au.csiro.pathling.sql.PathlingFunctions; +import au.csiro.pathling.sql.SqlExpressions; import ca.uhn.fhir.context.FhirContext; import java.util.ArrayList; import java.util.Collection; @@ -115,7 +115,7 @@ public ResultWithExpressions buildQuery(@Nonnull final AggregateRequest query) { // Remove synthetic fields from struct values (such as _fid) before grouping. final DatasetWithColumnMap datasetWithNormalizedGroupings = createColumns( groupingsAndFilters, groupings.stream().map(FhirPath::getValueColumn) - .map(PathlingFunctions::pruneSyntheticFields).toArray(Column[]::new)); + .map(SqlExpressions::pruneSyntheticFields).toArray(Column[]::new)); groupingsAndFilters = datasetWithNormalizedGroupings.getDataset(); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java new file mode 100644 index 0000000000..d787a7362b --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath; + +import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import java.util.Optional; +import java.util.function.Function; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.functions; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; + +public interface Temporal { + + @Nonnull + Function getDateArithmeticOperation( + @Nonnull MathOperation operation, @Nonnull Dataset dataset, + @Nonnull String expression); + + @Nonnull + static Function buildDateArithmeticOperation( + @Nonnull final FhirPath source, final @Nonnull MathOperation operation, + final @Nonnull Dataset dataset, final @Nonnull String expression, + final String additionFunctionName, final String subtractionFunctionName) { + return target -> { + final String functionName; + final Optional eidColumn = NonLiteralPath.findEidColumn(source, target); + final Optional thisColumn = NonLiteralPath.findThisColumn(source, target); + + switch (operation) { + case ADDITION: + functionName = additionFunctionName; + break; + case SUBTRACTION: + functionName = subtractionFunctionName; + break; + default: + throw new AssertionError("Unsupported date arithmetic operation: " + operation); + } + + final Column valueColumn = functions.callUDF(functionName, source.getValueColumn(), + target.getValueColumn()); + return ElementPath.build(expression, dataset, source.getIdColumn(), eidColumn, valueColumn, + true, + Optional.empty(), thisColumn, FHIRDefinedType.DATETIME); + }; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java index 32b73496dd..b25028b6da 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java @@ -6,13 +6,19 @@ package au.csiro.pathling.fhirpath.element; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static org.apache.spark.sql.functions.to_timestamp; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.sql.dates.AddDurationToDate; +import au.csiro.pathling.sql.dates.SubtractDurationFromDate; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -33,7 +39,8 @@ * @author John Grimes */ @Slf4j -public class DatePath extends ElementPath implements Materializable, Comparable { +public class DatePath extends ElementPath implements Materializable, Comparable, + Temporal { private static final ThreadLocal FULL_DATE_FORMAT = ThreadLocal .withInitial(() -> { @@ -134,4 +141,14 @@ public boolean isComparableTo(@Nonnull final Class type) { public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof DateLiteralPath; } + + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToDate.FUNCTION_NAME, SubtractDurationFromDate.FUNCTION_NAME); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index e741b16d3c..afe1000e4e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -6,17 +6,24 @@ package au.csiro.pathling.fhirpath.element; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static org.apache.spark.sql.functions.to_timestamp; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.sql.dates.AddDurationToDateTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; import com.google.common.collect.ImmutableSet; import java.text.SimpleDateFormat; +import java.util.List; import java.util.Optional; import java.util.TimeZone; import java.util.function.BiFunction; @@ -36,15 +43,25 @@ * @author John Grimes */ public class DateTimePath extends ElementPath implements Materializable, - Comparable { + Comparable, Temporal { private static final TimeZone TIME_ZONE = TimeZone.getTimeZone("GMT"); - private static final ThreadLocal DATE_FORMAT = ThreadLocal - .withInitial(() -> { - final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); - format.setTimeZone(TIME_ZONE); - return format; - }); + private static final ThreadLocal FULL_DATE_FORMAT = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + private static final ThreadLocal SECONDS_DATE_FORMAT = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); + private static final ThreadLocal MINUTES_DATE_FORMAT = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mmXXX"); + private static final ThreadLocal HOURS_DATE_FORMAT = + dateFormatThreadLocal("yyyy-MM-dd'T'HHXXX"); + private static final ThreadLocal FULL_DATE_FORMAT_NO_TZ = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + private static final ThreadLocal SECONDS_DATE_FORMAT_NO_TZ = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); + private static final ThreadLocal MINUTES_DATE_FORMAT_NO_TZ = + dateFormatThreadLocal("yyyy-MM-dd'T'HH:mmXXX"); + private static final ThreadLocal HOURS_DATE_FORMAT_NO_TZ = + dateFormatThreadLocal("yyyy-MM-dd'T'HHXXX"); private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(DatePath.class, DateTimePath.class, DateLiteralPath.class, DateTimeLiteralPath.class, @@ -109,8 +126,14 @@ public static Function buildComparison(@Nonnull final Compar .apply(to_timestamp(source.getValueColumn()), to_timestamp(target.getValueColumn())); } - public static SimpleDateFormat getDateFormat() { - return DATE_FORMAT.get(); + public static SimpleDateFormat getDefaultDateFormat() { + return FULL_DATE_FORMAT.get(); + } + + public static List getAllDateFormats() { + return List.of(FULL_DATE_FORMAT.get(), SECONDS_DATE_FORMAT.get(), MINUTES_DATE_FORMAT.get(), + HOURS_DATE_FORMAT.get(), FULL_DATE_FORMAT_NO_TZ.get(), SECONDS_DATE_FORMAT_NO_TZ.get(), + MINUTES_DATE_FORMAT_NO_TZ.get(), HOURS_DATE_FORMAT_NO_TZ.get()); } public static TimeZone getTimeZone() { @@ -137,4 +160,24 @@ public boolean isComparableTo(@Nonnull final Class type) { public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof DateTimeLiteralPath; } + + @Nonnull + private static ThreadLocal dateFormatThreadLocal(final String pattern) { + return ThreadLocal + .withInitial(() -> { + final SimpleDateFormat format = new SimpleDateFormat(pattern); + format.setTimeZone(TIME_ZONE); + return format; + }); + } + + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToDateTime.FUNCTION_NAME, SubtractDurationFromDateTime.FUNCTION_NAME); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java index fe702869e1..383db572d1 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java @@ -23,7 +23,7 @@ import au.csiro.pathling.fhirpath.literal.StringLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.sql.MapperWithPreview; -import au.csiro.pathling.sql.SqlExtensions; +import au.csiro.pathling.sql.SqlOperations; import java.util.List; import java.util.Set; import javax.annotation.Nonnull; @@ -91,7 +91,7 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { // This de-duplicates the Codings to be validated, then performs the validation on a // per-partition basis. - final Dataset resultDataset = SqlExtensions + final Dataset resultDataset = SqlOperations .mapWithPartitionPreview(dataset, codingArrayCol, SimpleCodingsDecoders::decodeList, mapper, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/subsumes/SubsumesFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/subsumes/SubsumesFunction.java index 9ab257ccfc..df3eea0007 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/subsumes/SubsumesFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/subsumes/SubsumesFunction.java @@ -24,7 +24,7 @@ import au.csiro.pathling.fhirpath.function.NamedFunction; import au.csiro.pathling.fhirpath.function.NamedFunctionInput; import au.csiro.pathling.fhirpath.parser.ParserContext; -import au.csiro.pathling.sql.SqlExtensions; +import au.csiro.pathling.sql.SqlOperations; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.Column; @@ -111,7 +111,7 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { input.getContext().getTerminologyServiceFactory().get(), inverted); - final Dataset resultDataset = SqlExtensions + final Dataset resultDataset = SqlOperations .mapWithPartitionPreview(idAndCodingSet, codingPairCol, SimpleCodingsDecoders::decodeListPair, mapper, StructField.apply("result", DataTypes.BooleanType, true, Metadata.empty())); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java index 4a291ea506..9f1be3bcab 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java @@ -29,7 +29,7 @@ import au.csiro.pathling.fhirpath.literal.StringLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.sql.MapperWithPreview; -import au.csiro.pathling.sql.SqlExtensions; +import au.csiro.pathling.sql.SqlOperations; import au.csiro.pathling.terminology.ConceptTranslator; import au.csiro.pathling.utilities.Strings; import java.util.List; @@ -168,7 +168,7 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { conceptMapUrl, reverse, Strings.parseCsvList(equivalence, wrapInUserInputError(ConceptMapEquivalence::fromCode))); - final Dataset translatedDataset = SqlExtensions + final Dataset translatedDataset = SqlOperations .mapWithPartitionPreview(dataset, codingArrayCol, SimpleCodingsDecoders::decodeList, mapper, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java index 6126befe9f..90068295ab 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java @@ -6,14 +6,19 @@ package au.csiro.pathling.fhirpath.literal; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static au.csiro.pathling.utilities.Preconditions.check; import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.DatePath; import au.csiro.pathling.fhirpath.element.DateTimePath; +import au.csiro.pathling.sql.dates.AddDurationToDate; +import au.csiro.pathling.sql.dates.SubtractDurationFromDate; import java.text.ParseException; import java.util.Date; import java.util.Optional; @@ -30,7 +35,8 @@ * * @author John Grimes */ -public class DateLiteralPath extends LiteralPath implements Materializable, Comparable { +public class DateLiteralPath extends LiteralPath implements Materializable, Comparable, + Temporal { @Nonnull private Optional format; @@ -130,6 +136,15 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof DatePath; } + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToDate.FUNCTION_NAME, SubtractDurationFromDate.FUNCTION_NAME); + } + private enum DateLiteralFormat { FULL, YEAR_MONTH_DATE, YEAR_ONLY } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java index 79c12de529..707fa50830 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java @@ -6,14 +6,21 @@ package au.csiro.pathling.fhirpath.literal; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static au.csiro.pathling.utilities.Preconditions.check; +import static au.csiro.pathling.utilities.Preconditions.checkNotNull; import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.DateTimePath; +import au.csiro.pathling.sql.dates.AddDurationToDateTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Date; import java.util.Optional; import java.util.function.Function; @@ -32,7 +39,7 @@ * @author John Grimes */ public class DateTimeLiteralPath extends LiteralPath implements Materializable, - Comparable { + Comparable, Temporal { @SuppressWarnings("WeakerAccess") protected DateTimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @@ -53,16 +60,31 @@ protected DateTimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull fina public static DateTimeLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) throws ParseException { final String dateTimeString = fhirPath.replaceFirst("^@", ""); - final java.util.Date date = DateTimePath.getDateFormat().parse(dateTimeString); + final java.util.Date date = parseDateTimeString(dateTimeString); final DateTimeType literalValue = new DateTimeType(date); literalValue.setTimeZone(DateTimePath.getTimeZone()); return new DateTimeLiteralPath(context.getDataset(), context.getIdColumn(), literalValue); } + @Nonnull + private static Date parseDateTimeString(@Nonnull final String dateTimeString) + throws ParseException { + ParseException parseException = null; + for (final SimpleDateFormat format : DateTimePath.getAllDateFormats()) { + try { + return format.parse(dateTimeString); + } catch (final ParseException e) { + parseException = e; + } + } + checkNotNull(parseException); + throw parseException; + } + @Nonnull @Override public String getExpression() { - return "@" + DateTimePath.getDateFormat().format(getLiteralValue().getValue()); + return "@" + DateTimePath.getDefaultDateFormat().format(getLiteralValue().getValue()); } @Override @@ -104,5 +126,13 @@ public Optional getValueFromRow(@Nonnull final Row row, public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof DateTimePath; } - + + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToDateTime.FUNCTION_NAME, SubtractDurationFromDateTime.FUNCTION_NAME); + } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java new file mode 100644 index 0000000000..b13856bf52 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.QueryHelpers.join; +import static au.csiro.pathling.fhirpath.operator.Operator.buildExpression; +import static au.csiro.pathling.utilities.Preconditions.checkUserInput; + +import au.csiro.pathling.QueryHelpers.JoinType; +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; + +/** + * Provides the functionality of the family of math operators within FHIRPath, i.e. +, -, *, / and + * mod. + * + * @author John Grimes + * @see Math + */ +public class DateArithmeticOperator implements Operator { + + @Nonnull + private final MathOperation type; + + /** + * @param type The type of math operation + */ + public DateArithmeticOperator(@Nonnull final MathOperation type) { + this.type = type; + } + + @Nonnull + @Override + public FhirPath invoke(@Nonnull final OperatorInput input) { + final FhirPath left = input.getLeft(); + final FhirPath right = input.getRight(); + checkUserInput(left instanceof Temporal, + type + " operator does not support left operand: " + left.getExpression()); + checkUserInput(right instanceof QuantityLiteralPath, + type + " operator does not support right operand: " + right.getExpression()); + final QuantityLiteralPath calendarDuration = (QuantityLiteralPath) right; + checkUserInput(calendarDuration.getJavaValue().getSystem() + .equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI), + "Right operand of " + type + " operator must be a calendar duration"); + checkUserInput(left.isSingular(), + "Left operand to " + type + " operator must be singular: " + left.getExpression()); + checkUserInput(right.isSingular(), + "Right operand to " + type + " operator must be singular: " + right.getExpression()); + + final Temporal temporal = (Temporal) left; + final String expression = buildExpression(input, type.toString()); + final Dataset dataset = join(input.getContext(), left, right, JoinType.LEFT_OUTER); + + return temporal.getDateArithmeticOperation(type, dataset, expression) + .apply(calendarDuration); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java index 4a00ef2dff..a1ed24eef6 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java @@ -14,6 +14,8 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import javax.annotation.Nonnull; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -42,6 +44,12 @@ public MathOperator(@Nonnull final MathOperation type) { public FhirPath invoke(@Nonnull final OperatorInput input) { final FhirPath left = input.getLeft(); final FhirPath right = input.getRight(); + + // Check whether this needs to be delegated off to the DateArithmeticOperator. + if (left instanceof Temporal && right instanceof QuantityLiteralPath) { + return new DateArithmeticOperator(type).invoke(input); + } + checkUserInput(left instanceof Numeric, type + " operator does not support left operand: " + left.getExpression()); checkUserInput(right instanceof Numeric, diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index 841ea07f12..eba2939125 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -9,17 +9,16 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.Configuration.Storage.Aws; import au.csiro.pathling.async.SparkListener; -import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import au.csiro.pathling.sql.PathlingStrategy; -import au.csiro.pathling.terminology.CodingToLiteral; -import au.csiro.pathling.terminology.ucum.ComparableQuantity; +import au.csiro.pathling.sql.SqlStrategy; +import au.csiro.pathling.sql.udf.SqlFunction1; +import au.csiro.pathling.sql.udf.SqlFunction2; import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.Optional; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.types.DataTypes; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Profile; @@ -53,7 +52,8 @@ public class Spark { public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, - @Nonnull final ComparableQuantity comparableQuantity) { + @Nonnull final List sqlFunction1, + @Nonnull final List sqlFunction2) { log.debug("Creating Spark session"); resolveSparkConfiguration(environment); @@ -62,13 +62,14 @@ public static SparkSession build(@Nonnull final Configuration configuration, .getOrCreate(); sparkListener.ifPresent(l -> spark.sparkContext().addSparkListener(l)); - // Configure user defined functions. - PathlingStrategy.setup(spark); - spark.udf() - .register(CodingToLiteral.FUNCTION_NAME, new CodingToLiteral(), DataTypes.StringType); - spark.udf() - .register(ComparableQuantity.FUNCTION_NAME, comparableQuantity, - QuantityEncoding.dataType()); + // Configure user defined strategy and functions. + SqlStrategy.setup(spark); + for (final SqlFunction1 function : sqlFunction1) { + spark.udf().register(function.getName(), function, function.getReturnType()); + } + for (final SqlFunction2 function : sqlFunction2) { + spark.udf().register(function.getName(), function, function.getReturnType()); + } // Configure AWS driver and credentials. configureAwsDriver(configuration, spark); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/PathlingFunctions.java b/fhir-server/src/main/java/au/csiro/pathling/sql/SqlExpressions.java similarity index 95% rename from fhir-server/src/main/java/au/csiro/pathling/sql/PathlingFunctions.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/SqlExpressions.java index 1e6ecdf7cd..9155cd2156 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/PathlingFunctions.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/SqlExpressions.java @@ -12,7 +12,7 @@ /** * Pathling specific SQL functions. */ -public interface PathlingFunctions { +public interface SqlExpressions { /** * A function that removes all fields starting with '_' (underscore) from struct values. Other diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/SqlExtensions.java b/fhir-server/src/main/java/au/csiro/pathling/sql/SqlOperations.java similarity index 98% rename from fhir-server/src/main/java/au/csiro/pathling/sql/SqlExtensions.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/SqlOperations.java index 9abdb92090..38bf21b03f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/SqlExtensions.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/SqlOperations.java @@ -16,7 +16,7 @@ /** * Custom dataset operations. */ -public interface SqlExtensions { +public interface SqlOperations { /** * Creates a new {@link Dataset} with the additional column as specified in {@code resultField} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java new file mode 100644 index 0000000000..3dfa42712e --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.time.LocalDate; +import java.util.function.BiFunction; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class AddDurationToDate extends DateArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = -5029179160644275584L; + + public static final String FUNCTION_NAME = "add_duration_to_date"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performAddition; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java new file mode 100644 index 0000000000..48c473458f --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.time.ZonedDateTime; +import java.util.function.BiFunction; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class AddDurationToDateTime extends DateTimeArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = 6922227603585641053L; + + public static final String FUNCTION_NAME = "add_duration_to_datetime"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performAddition; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java new file mode 100644 index 0000000000..1aba49e9f3 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.fhirpath.element.DateTimePath; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Date; +import java.util.function.Function; +import org.hl7.fhir.r4.model.DateType; + +public abstract class DateArithmeticFunction extends TemporalArithmeticFunction { + + private static final long serialVersionUID = 6759548804191034570L; + + @Override + Function parseEncodedValue() { + return LocalDate::parse; + } + + @Override + Function encodeResult() { + return (resultDate) -> new DateType( + new Date(resultDate.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) * 1000)) + .setTimeZone(DateTimePath.getTimeZone()) + .getValueAsString(); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java new file mode 100644 index 0000000000..8f0a9bc479 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.fhirpath.element.DateTimePath; +import java.time.ZonedDateTime; +import java.util.Date; +import java.util.function.Function; +import org.hl7.fhir.r4.model.DateTimeType; + +public abstract class DateTimeArithmeticFunction extends TemporalArithmeticFunction { + + private static final long serialVersionUID = -6669722492626320119L; + + @Override + Function parseEncodedValue() { + return ZonedDateTime::parse; + } + + @Override + Function encodeResult() { + return (resultDateTime) -> new DateTimeType(new Date(resultDateTime.toEpochSecond() * 1000)) + .setTimeZone(DateTimePath.getTimeZone()) + .getValueAsString(); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java new file mode 100644 index 0000000000..7566472591 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.time.LocalDate; +import java.util.function.BiFunction; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class SubtractDurationFromDate extends DateArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = 5201879133976866457L; + + public static final String FUNCTION_NAME = "subtract_duration_from_date"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performSubtraction; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java new file mode 100644 index 0000000000..bd0463bdf1 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import java.time.ZonedDateTime; +import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class SubtractDurationFromDateTime extends DateTimeArithmeticFunction { + + private static final long serialVersionUID = -5922228168177608861L; + + public static final String FUNCTION_NAME = "subtract_duration_from_datetime"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performSubtraction; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java new file mode 100644 index 0000000000..db701415b6 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -0,0 +1,109 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.sql.udf.SqlFunction2; +import com.google.common.collect.ImmutableMap; +import java.math.RoundingMode; +import java.time.temporal.ChronoUnit; +import java.time.temporal.Temporal; +import java.time.temporal.TemporalUnit; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Quantity; + +public abstract class TemporalArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = -5016153440496309996L; + + static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("year", ChronoUnit.YEARS) + .put("years", ChronoUnit.YEARS) + .put("month", ChronoUnit.MONTHS) + .put("months", ChronoUnit.MONTHS) + .put("week", ChronoUnit.WEEKS) + .put("weeks", ChronoUnit.WEEKS) + .put("day", ChronoUnit.DAYS) + .put("days", ChronoUnit.DAYS) + .put("hour", ChronoUnit.HOURS) + .put("hours", ChronoUnit.HOURS) + .put("minute", ChronoUnit.MINUTES) + .put("minutes", ChronoUnit.MINUTES) + .put("second", ChronoUnit.SECONDS) + .put("seconds", ChronoUnit.SECONDS) + .put("millisecond", ChronoUnit.MILLIS) + .put("milliseconds", ChronoUnit.MILLIS) + .build(); + + @Nonnull + protected IntermediateType performAddition(@Nonnull final IntermediateType temporal, + @Nonnull final Quantity calendarDuration) { + if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { + throw new IllegalArgumentException("Calendar duration must have a system of " + + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); + } + final long amountToAdd = calendarDuration.getValue() + .setScale(0, RoundingMode.HALF_UP) + .longValue(); + final TemporalUnit temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( + calendarDuration.getCode()); + + //noinspection unchecked + return (IntermediateType) temporal.plus(amountToAdd, temporalUnit); + } + + @Nonnull + protected IntermediateType performSubtraction(@Nonnull final IntermediateType temporal, + @Nonnull final Quantity calendarDuration) { + if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { + throw new IllegalArgumentException("Calendar duration must have a system of " + + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); + } + final long amountToAdd = calendarDuration.getValue() + .setScale(0, RoundingMode.HALF_UP) + .longValue(); + final TemporalUnit temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( + calendarDuration.getCode()); + + //noinspection unchecked + return (IntermediateType) temporal.minus(amountToAdd, temporalUnit); + } + + abstract Function parseEncodedValue(); + + abstract BiFunction getOperationFunction(); + + abstract Function encodeResult(); + + @Override + public DataType getReturnType() { + return DataTypes.StringType; + } + + @Nullable + @Override + public String call(@Nullable final String temporalValue, @Nullable final Row calendarDurationRow) + throws Exception { + if (temporalValue == null || calendarDurationRow == null) { + return null; + } + final IntermediateType temporal = parseEncodedValue().apply(temporalValue); + final Quantity calendarDuration = QuantityEncoding.decode(calendarDurationRow); + final IntermediateType result = getOperationFunction().apply(temporal, calendarDuration); + return encodeResult().apply(result); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java new file mode 100644 index 0000000000..3d14307d90 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java @@ -0,0 +1,18 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.udf; + +import org.apache.spark.sql.api.java.UDF1; +import org.apache.spark.sql.types.DataType; + +public interface SqlFunction1 extends UDF1 { + + String getName(); + + DataType getReturnType(); + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java new file mode 100644 index 0000000000..ce26dba6f2 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java @@ -0,0 +1,18 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.udf; + +import org.apache.spark.sql.api.java.UDF2; +import org.apache.spark.sql.types.DataType; + +public interface SqlFunction2 extends UDF2 { + + String getName(); + + DataType getReturnType(); + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java index 4248380d48..4c57cd0f86 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java @@ -8,17 +8,23 @@ import au.csiro.pathling.fhirpath.encoding.CodingEncoding; import au.csiro.pathling.fhirpath.literal.CodingLiteral; +import au.csiro.pathling.sql.udf.SqlFunction1; import javax.annotation.Nullable; import org.apache.spark.sql.Row; -import org.apache.spark.sql.api.java.UDF1; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; import org.hl7.fhir.r4.model.Coding; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; /** * Spark UDF to convert a Coding struct to a valid Coding literal string. * * @author John Grimes */ -public class CodingToLiteral implements UDF1 { +@Component +@Profile("core") +public class CodingToLiteral implements SqlFunction1 { /** * The name of this function when used within SQL. @@ -27,6 +33,16 @@ public class CodingToLiteral implements UDF1 { private static final long serialVersionUID = -6274263255779613070L; + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + public DataType getReturnType() { + return DataTypes.StringType; + } + @Override @Nullable public String call(@Nullable final Row row) throws Exception { diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index 5ce5fef3bd..3b93c4038e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -9,6 +9,7 @@ import au.csiro.pathling.fhirpath.element.DecimalPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.sql.udf.SqlFunction1; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; import java.util.Map; @@ -16,17 +17,19 @@ import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.Row; -import org.apache.spark.sql.api.java.UDF1; +import org.apache.spark.sql.types.DataType; import org.fhir.ucum.Decimal; import org.fhir.ucum.Pair; import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component +@Profile("core") @Slf4j -public class ComparableQuantity implements UDF1 { +public class ComparableQuantity implements SqlFunction1 { @Nonnull private final UcumService ucumService; @@ -48,6 +51,16 @@ public ComparableQuantity(@Nonnull final UcumService ucumService) { this.ucumService = ucumService; } + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + public DataType getReturnType() { + return QuantityEncoding.dataType(); + } + @Nullable @Override public Row call(@Nullable final Row row) throws Exception { diff --git a/fhir-server/src/main/scala/au/csiro/pathling/sql/PathlingStrategy.scala b/fhir-server/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala similarity index 71% rename from fhir-server/src/main/scala/au/csiro/pathling/sql/PathlingStrategy.scala rename to fhir-server/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala index 4315664d98..c55dc6f0d9 100644 --- a/fhir-server/src/main/scala/au/csiro/pathling/sql/PathlingStrategy.scala +++ b/fhir-server/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala @@ -13,7 +13,7 @@ import org.apache.spark.sql.execution._ /** * Custom spark SQL strategy with additional rules for custom Pathling operations. */ -object PathlingStrategy extends Strategy { +object SqlStrategy extends Strategy { override def apply(plan: LogicalPlan): Seq[SparkPlan] = { plan match { @@ -26,13 +26,13 @@ object PathlingStrategy extends Strategy { /** - * Injects PathlingStrategy into a given Spark session. + * Injects SqlStrategy into a given Spark session. * - * @param session Spark session to add PathlingStrategy to + * @param session Spark session to add SqlStrategy to */ def setup(session: SparkSession): Unit = { - if (!session.experimental.extraStrategies.contains(PathlingStrategy)) { - session.experimental.extraStrategies = Seq(PathlingStrategy) ++ session.experimental.extraStrategies + if (!session.experimental.extraStrategies.contains(SqlStrategy)) { + session.experimental.extraStrategies = Seq(SqlStrategy) ++ session.experimental.extraStrategies } } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index cd8a578cfd..cad821700d 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -51,7 +51,6 @@ public class DateArithmeticTest { FhirContext fhirContext; static final String ID_ALIAS = "_abc123"; - static final List OPERATORS = List.of("+", "-"); @Value static class TestParameters { @@ -89,7 +88,7 @@ Stream parameters() throws ParseException { .withIdColumn(ID_ALIAS) .withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T13:28:17-05:00") - .withRow("patient-2", "2017-01-01T00:00:00.000Z") + .withRow("patient-2", "2017-01-01T00:00:00+00:00") .withRow("patient-3", "2025-06-21T00:15:00+10:00") .build(); final ElementPath dateTimePath = new ElementPathBuilder(spark) @@ -114,7 +113,7 @@ Stream parameters() throws ParseException { .build(); final DateTimeLiteralPath dateTimeLiteral = DateTimeLiteralPath.fromString( - "@2015-02-07T18:28:17.000Z", dateTimePath); + "@2015-02-07T18:28:17+00:00", dateTimePath); final DateLiteralPath dateLiteral = DateLiteralPath.fromString("@2015-02-07", datePath); final ParserContext context = new ParserContextBuilder(spark, fhirContext) @@ -125,10 +124,10 @@ Stream parameters() throws ParseException { parameters.addAll(dateTimeSubtraction(dateTimePath, context)); parameters.addAll(dateAddition(datePath, context)); parameters.addAll(dateSubtraction(datePath, context)); - parameters.addAll(dateTimeLiteralAddition(datePath, context)); - parameters.addAll(dateTimeLiteralSubtraction(datePath, context)); - parameters.addAll(dateLiteralAddition(datePath, context)); - parameters.addAll(dateLiteralSubtraction(datePath, context)); + parameters.addAll(dateTimeLiteralAddition(dateTimeLiteral, context)); + parameters.addAll(dateTimeLiteralSubtraction(dateTimeLiteral, context)); + parameters.addAll(dateLiteralAddition(dateLiteral, context)); + parameters.addAll(dateLiteralSubtraction(dateLiteral, context)); return parameters.stream(); } @@ -140,9 +139,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000Z") - .withRow("patient-2", "2027-01-01T00:00:00.000Z") - .withRow("patient-3", "2035-06-20T14:15:00.000Z") + .withRow("patient-1", "2025-02-07T18:28:17+00:00") + .withRow("patient-2", "2027-01-01T00:00:00+00:00") + .withRow("patient-3", "2035-06-20T14:15:00+00:00") .build()) ); @@ -150,9 +149,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000Z") - .withRow("patient-2", "2017-10-01T00:00:00.000Z") - .withRow("patient-3", "2026-03-20T14:15:00.000Z") + .withRow("patient-1", "2015-11-07T18:28:17+00:00") + .withRow("patient-2", "2017-10-01T00:00:00+00:00") + .withRow("patient-3", "2026-03-20T14:15:00+00:00") .build()) ); @@ -160,9 +159,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000Z") - .withRow("patient-2", "2017-01-15T00:00:00.000Z") - .withRow("patient-3", "2025-07-04T14:15:00.000Z") + .withRow("patient-1", "2015-02-21T18:28:17+00:00") + .withRow("patient-2", "2017-01-15T00:00:00+00:00") + .withRow("patient-3", "2025-07-04T14:15:00+00:00") .build()) ); @@ -170,9 +169,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000Z") - .withRow("patient-2", "2017-01-31T00:00:00.000Z") - .withRow("patient-3", "2025-07-20T14:15:00.000Z") + .withRow("patient-1", "2015-03-09T18:28:17+00:00") + .withRow("patient-2", "2017-01-31T00:00:00+00:00") + .withRow("patient-3", "2025-07-20T14:15:00+00:00") .build()) ); @@ -180,9 +179,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000Z") - .withRow("patient-2", "2017-01-01T12:00:00.000Z") - .withRow("patient-3", "2025-06-21T12:15:00.000Z") + .withRow("patient-1", "2015-02-08T06:28:17+00:00") + .withRow("patient-2", "2017-01-01T12:00:00+00:00") + .withRow("patient-3", "2025-06-21T02:15:00+00:00") .build()) ); @@ -190,9 +189,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000Z") - .withRow("patient-2", "2017-01-01T00:30:00.000Z") - .withRow("patient-3", "2025-06-20T14:45:00.000Z") + .withRow("patient-1", "2015-02-07T18:58:17+00:00") + .withRow("patient-2", "2017-01-01T00:30:00+00:00") + .withRow("patient-3", "2025-06-20T14:45:00+00:00") .build()) ); @@ -200,21 +199,12 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000Z") - .withRow("patient-2", "2017-01-01T00:00:10.000Z") - .withRow("patient-3", "2025-06-20T14:15:10.000Z") + .withRow("patient-1", "2015-02-07T18:28:27+00:00") + .withRow("patient-2", "2017-01-01T00:00:10+00:00") + .withRow("patient-3", "2025-06-20T14:15:10+00:00") .build()) ); - parameters.add(new TestParameters("DateTime + 300 milliseconds", dateTimePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300Z") - .withRow("patient-2", "2017-01-01T00:00:00.300Z") - .withRow("patient-3", "2025-06-20T14:15:00.300Z") - .build()) - ); return parameters; } @@ -225,9 +215,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000Z") - .withRow("patient-2", "2007-01-01T00:00:00.000Z") - .withRow("patient-3", "2015-06-20T14:15:00.000Z") + .withRow("patient-1", "2005-02-07T18:28:17+00:00") + .withRow("patient-2", "2007-01-01T00:00:00+00:00") + .withRow("patient-3", "2015-06-20T14:15:00+00:00") .build()) ); @@ -235,9 +225,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000Z") - .withRow("patient-2", "2016-04-01T00:00:00.000Z") - .withRow("patient-3", "2024-09-20T14:15:00.000Z") + .withRow("patient-1", "2014-05-07T18:28:17+00:00") + .withRow("patient-2", "2016-04-01T00:00:00+00:00") + .withRow("patient-3", "2024-09-20T14:15:00+00:00") .build()) ); @@ -245,9 +235,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000Z") - .withRow("patient-2", "2016-12-18T00:00:00.000Z") - .withRow("patient-3", "2025-06-06T14:15:00.000Z") + .withRow("patient-1", "2015-01-24T18:28:17+00:00") + .withRow("patient-2", "2016-12-18T00:00:00+00:00") + .withRow("patient-3", "2025-06-06T14:15:00+00:00") .build()) ); @@ -255,9 +245,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000Z") - .withRow("patient-2", "2016-12-02T00:00:00.000Z") - .withRow("patient-3", "2025-05-21T14:15:00.000Z") + .withRow("patient-1", "2015-01-08T18:28:17+00:00") + .withRow("patient-2", "2016-12-02T00:00:00+00:00") + .withRow("patient-3", "2025-05-21T14:15:00+00:00") .build()) ); @@ -265,9 +255,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000Z") - .withRow("patient-2", "2016-12-31T12:00:00.000Z") - .withRow("patient-3", "2025-06-20T02:15:00.000Z") + .withRow("patient-1", "2015-02-07T06:28:17+00:00") + .withRow("patient-2", "2016-12-31T12:00:00+00:00") + .withRow("patient-3", "2025-06-20T02:15:00+00:00") .build()) ); @@ -275,9 +265,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000Z") - .withRow("patient-2", "2016-12-31T23:30:00.000Z") - .withRow("patient-3", "2025-06-20T13:45:00.000Z") + .withRow("patient-1", "2015-02-07T17:58:17+00:00") + .withRow("patient-2", "2016-12-31T23:30:00+00:00") + .withRow("patient-3", "2025-06-20T13:45:00+00:00") .build()) ); @@ -285,21 +275,12 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000Z") - .withRow("patient-2", "2016-12-31T23:59:50.000Z") - .withRow("patient-3", "2025-06-20T14:14:50.000Z") + .withRow("patient-1", "2015-02-07T18:28:07+00:00") + .withRow("patient-2", "2016-12-31T23:59:50+00:00") + .withRow("patient-3", "2025-06-20T14:14:50+00:00") .build()) ); - parameters.add(new TestParameters("DateTime - 300 milliseconds", dateTimePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700Z") - .withRow("patient-2", "2016-12-31T23:59:59.700Z") - .withRow("patient-3", "2025-06-20T14:14:59.700Z") - .build()) - ); return parameters; } @@ -310,9 +291,9 @@ Collection dateAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000Z") - .withRow("patient-2", "2027-01-01T00:00:00.000Z") - .withRow("patient-3", "2035-06-20T14:15:00.000Z") + .withRow("patient-1", "2025-02-07") + .withRow("patient-2", "2027-01-01") + .withRow("patient-3", "2035-06-21") .build()) ); @@ -320,9 +301,9 @@ Collection dateAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000Z") - .withRow("patient-2", "2017-10-01T00:00:00.000Z") - .withRow("patient-3", "2026-03-20T14:15:00.000Z") + .withRow("patient-1", "2015-11-07") + .withRow("patient-2", "2017-10-01") + .withRow("patient-3", "2026-03-21") .build()) ); @@ -330,9 +311,9 @@ Collection dateAddition( QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000Z") - .withRow("patient-2", "2017-01-15T00:00:00.000Z") - .withRow("patient-3", "2025-07-04T14:15:00.000Z") + .withRow("patient-1", "2015-02-21") + .withRow("patient-2", "2017-01-15") + .withRow("patient-3", "2025-07-05") .build()) ); @@ -340,51 +321,12 @@ Collection dateAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000Z") - .withRow("patient-2", "2017-01-31T00:00:00.000Z") - .withRow("patient-3", "2025-07-20T14:15:00.000Z") + .withRow("patient-1", "2015-03-09") + .withRow("patient-2", "2017-01-31") + .withRow("patient-3", "2025-07-21") .build()) ); - parameters.add(new TestParameters("Date + 12 hours", datePath, - QuantityLiteralPath.fromCalendarDurationString("12 hours", datePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000Z") - .withRow("patient-2", "2017-01-01T12:00:00.000Z") - .withRow("patient-3", "2025-06-21T12:15:00.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date + 30 minutes", datePath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", datePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000Z") - .withRow("patient-2", "2017-01-01T00:30:00.000Z") - .withRow("patient-3", "2025-06-20T14:45:00.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date + 10 seconds", datePath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", datePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000Z") - .withRow("patient-2", "2017-01-01T00:00:10.000Z") - .withRow("patient-3", "2025-06-20T14:15:10.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date + 300 milliseconds", datePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", datePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300Z") - .withRow("patient-2", "2017-01-01T00:00:00.300Z") - .withRow("patient-3", "2025-06-20T14:15:00.300Z") - .build()) - ); return parameters; } @@ -395,9 +337,9 @@ Collection dateSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000Z") - .withRow("patient-2", "2007-01-01T00:00:00.000Z") - .withRow("patient-3", "2015-06-20T14:15:00.000Z") + .withRow("patient-1", "2005-02-07") + .withRow("patient-2", "2007-01-01") + .withRow("patient-3", "2015-06-21") .build()) ); @@ -405,9 +347,9 @@ Collection dateSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000Z") - .withRow("patient-2", "2016-04-01T00:00:00.000Z") - .withRow("patient-3", "2024-09-20T14:15:00.000Z") + .withRow("patient-1", "2014-05-07") + .withRow("patient-2", "2016-04-01") + .withRow("patient-3", "2024-09-21") .build()) ); @@ -415,9 +357,9 @@ Collection dateSubtraction( QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000Z") - .withRow("patient-2", "2016-12-18T00:00:00.000Z") - .withRow("patient-3", "2025-06-06T14:15:00.000Z") + .withRow("patient-1", "2015-01-24") + .withRow("patient-2", "2016-12-18") + .withRow("patient-3", "2025-06-07") .build()) ); @@ -425,399 +367,264 @@ Collection dateSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000Z") - .withRow("patient-2", "2016-12-02T00:00:00.000Z") - .withRow("patient-3", "2025-05-21T14:15:00.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date - 12 hours", datePath, - QuantityLiteralPath.fromCalendarDurationString("12 hours", datePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000Z") - .withRow("patient-2", "2016-12-31T12:00:00.000Z") - .withRow("patient-3", "2025-06-20T02:15:00.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date - 30 minutes", datePath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", datePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000Z") - .withRow("patient-2", "2016-12-31T23:30:00.000Z") - .withRow("patient-3", "2025-06-20T13:45:00.000Z") + .withRow("patient-1", "2015-01-08") + .withRow("patient-2", "2016-12-02") + .withRow("patient-3", "2025-05-22") .build()) ); - parameters.add(new TestParameters("Date - 10 seconds", datePath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", datePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000Z") - .withRow("patient-2", "2016-12-31T23:59:50.000Z") - .withRow("patient-3", "2025-06-20T14:14:50.000Z") - .build()) - ); - - parameters.add(new TestParameters("Date - 300 milliseconds", datePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", datePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700Z") - .withRow("patient-2", "2016-12-31T23:59:59.700Z") - .withRow("patient-3", "2025-06-20T14:14:59.700Z") - .build()) - ); return parameters; } Collection dateTimeLiteralAddition( final FhirPath dateTimeLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 10 years", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000Z") - .withRow("patient-2", "2025-02-07T18:28:17.000Z") - .withRow("patient-3", "2025-02-07T18:28:17.000Z") + .withRow("patient-1", "2025-02-07T18:28:17+00:00") + .withRow("patient-2", "2025-02-07T18:28:17+00:00") + .withRow("patient-3", "2025-02-07T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 9 months", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000Z") - .withRow("patient-2", "2015-11-07T18:28:17.000Z") - .withRow("patient-3", "2015-11-07T18:28:17.000Z") + .withRow("patient-1", "2015-11-07T18:28:17+00:00") + .withRow("patient-2", "2015-11-07T18:28:17+00:00") + .withRow("patient-3", "2015-11-07T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 2 weeks", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000Z") - .withRow("patient-2", "2015-02-21T18:28:17.000Z") - .withRow("patient-3", "2015-02-21T18:28:17.000Z") + .withRow("patient-1", "2015-02-21T18:28:17+00:00") + .withRow("patient-2", "2015-02-21T18:28:17+00:00") + .withRow("patient-3", "2015-02-21T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 30 days", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000Z") - .withRow("patient-2", "2015-03-09T18:28:17.000Z") - .withRow("patient-3", "2015-03-09T18:28:17.000Z") + .withRow("patient-1", "2015-03-09T18:28:17+00:00") + .withRow("patient-2", "2015-03-09T18:28:17+00:00") + .withRow("patient-3", "2015-03-09T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 12 hours", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000Z") - .withRow("patient-2", "2015-02-08T06:28:17.000Z") - .withRow("patient-3", "2015-02-08T06:28:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000Z") - .withRow("patient-2", "2015-02-07T18:58:17.000Z") - .withRow("patient-3", "2015-02-07T18:58:17.000Z") + .withRow("patient-1", "2015-02-08T06:28:17+00:00") + .withRow("patient-2", "2015-02-08T06:28:17+00:00") + .withRow("patient-3", "2015-02-08T06:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000Z") - .withRow("patient-2", "2015-02-07T18:28:27.000Z") - .withRow("patient-3", "2015-02-07T18:28:27.000Z") - .build()) + parameters.add( + new TestParameters("@2015-02-07T18:28:17+00:00 + 30 minutes", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), + context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:58:17+00:00") + .withRow("patient-2", "2015-02-07T18:58:17+00:00") + .withRow("patient-3", "2015-02-07T18:58:17+00:00") + .build()) ); parameters.add( - new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), + new TestParameters("@2015-02-07T18:28:17+00:00 + 10 seconds", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300Z") - .withRow("patient-2", "2015-02-07T18:28:17.300Z") - .withRow("patient-3", "2015-02-07T18:28:17.300Z") + .withRow("patient-1", "2015-02-07T18:28:27+00:00") + .withRow("patient-2", "2015-02-07T18:28:27+00:00") + .withRow("patient-3", "2015-02-07T18:28:27+00:00") .build()) ); + return parameters; } Collection dateTimeLiteralSubtraction( final FhirPath dateTimeLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 10 years", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000Z") - .withRow("patient-2", "2005-02-07T18:28:17.000Z") - .withRow("patient-3", "2005-02-07T18:28:17.000Z") + .withRow("patient-1", "2005-02-07T18:28:17+00:00") + .withRow("patient-2", "2005-02-07T18:28:17+00:00") + .withRow("patient-3", "2005-02-07T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 9 months", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000Z") - .withRow("patient-2", "2014-05-07T18:28:17.000Z") - .withRow("patient-3", "2014-05-07T18:28:17.000Z") + .withRow("patient-1", "2014-05-07T18:28:17+00:00") + .withRow("patient-2", "2014-05-07T18:28:17+00:00") + .withRow("patient-3", "2014-05-07T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 2 weeks", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000Z") - .withRow("patient-2", "2015-01-24T18:28:17.000Z") - .withRow("patient-3", "2015-01-24T18:28:17.000Z") + .withRow("patient-1", "2015-01-24T18:28:17+00:00") + .withRow("patient-2", "2015-01-24T18:28:17+00:00") + .withRow("patient-3", "2015-01-24T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 30 days", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000Z") - .withRow("patient-2", "2015-01-08T18:28:17.000Z") - .withRow("patient-3", "2015-01-08T18:28:17.000Z") + .withRow("patient-1", "2015-01-08T18:28:17+00:00") + .withRow("patient-2", "2015-01-08T18:28:17+00:00") + .withRow("patient-3", "2015-01-08T18:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateTimeLiteralPath, + parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 12 hours", dateTimeLiteralPath, QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000Z") - .withRow("patient-2", "2015-02-07T06:28:17.000Z") - .withRow("patient-3", "2015-02-07T06:28:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000Z") - .withRow("patient-2", "2015-02-07T17:58:17.000Z") - .withRow("patient-3", "2015-02-07T17:58:17.000Z") + .withRow("patient-1", "2015-02-07T06:28:17+00:00") + .withRow("patient-2", "2015-02-07T06:28:17+00:00") + .withRow("patient-3", "2015-02-07T06:28:17+00:00") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000Z") - .withRow("patient-2", "2015-02-07T18:28:07.000Z") - .withRow("patient-3", "2015-02-07T18:28:07.000Z") - .build()) + parameters.add( + new TestParameters("@2015-02-07T18:28:17+00:00 - 30 minutes", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimeLiteralPath), + context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T17:58:17+00:00") + .withRow("patient-2", "2015-02-07T17:58:17+00:00") + .withRow("patient-3", "2015-02-07T17:58:17+00:00") + .build()) ); parameters.add( - new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), + new TestParameters("@2015-02-07T18:28:17+00:00 - 10 seconds", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700Z") - .withRow("patient-2", "2015-02-07T18:28:16.700Z") - .withRow("patient-3", "2015-02-07T18:28:16.700Z") + .withRow("patient-1", "2015-02-07T18:28:07+00:00") + .withRow("patient-2", "2015-02-07T18:28:07+00:00") + .withRow("patient-3", "2015-02-07T18:28:07+00:00") .build()) ); + return parameters; } Collection dateLiteralAddition( final FhirPath dateLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 years", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 + 10 years", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("10 years", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000Z") - .withRow("patient-2", "2025-02-07T18:28:17.000Z") - .withRow("patient-3", "2025-02-07T18:28:17.000Z") + .withRow("patient-1", "2025-02-07") + .withRow("patient-2", "2025-02-07") + .withRow("patient-3", "2025-02-07") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 9 months", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 + 9 months", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("9 months", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000Z") - .withRow("patient-2", "2015-11-07T18:28:17.000Z") - .withRow("patient-3", "2015-11-07T18:28:17.000Z") + .withRow("patient-1", "2015-11-07") + .withRow("patient-2", "2015-11-07") + .withRow("patient-3", "2015-11-07") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 2 weeks", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 + 2 weeks", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000Z") - .withRow("patient-2", "2015-02-21T18:28:17.000Z") - .withRow("patient-3", "2015-02-21T18:28:17.000Z") + .withRow("patient-1", "2015-02-21") + .withRow("patient-2", "2015-02-21") + .withRow("patient-3", "2015-02-21") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 days", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 + 30 days", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000Z") - .withRow("patient-2", "2015-03-09T18:28:17.000Z") - .withRow("patient-3", "2015-03-09T18:28:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 12 hours", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("12 hours", dateLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000Z") - .withRow("patient-2", "2015-02-08T06:28:17.000Z") - .withRow("patient-3", "2015-02-08T06:28:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 30 minutes", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000Z") - .withRow("patient-2", "2015-02-07T18:58:17.000Z") - .withRow("patient-3", "2015-02-07T18:58:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z + 10 seconds", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000Z") - .withRow("patient-2", "2015-02-07T18:28:27.000Z") - .withRow("patient-3", "2015-02-07T18:28:27.000Z") + .withRow("patient-1", "2015-03-09") + .withRow("patient-2", "2015-03-09") + .withRow("patient-3", "2015-03-09") .build()) ); - parameters.add( - new TestParameters("@2015-02-07T18:28:17.000Z + 300 milliseconds", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateLiteralPath), - context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300Z") - .withRow("patient-2", "2015-02-07T18:28:17.300Z") - .withRow("patient-3", "2015-02-07T18:28:17.300Z") - .build()) - ); return parameters; } Collection dateLiteralSubtraction( final FhirPath dateLiteralPath, final ParserContext context) { final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 years", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 - 10 years", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("10 years", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000Z") - .withRow("patient-2", "2005-02-07T18:28:17.000Z") - .withRow("patient-3", "2005-02-07T18:28:17.000Z") + .withRow("patient-1", "2005-02-07") + .withRow("patient-2", "2005-02-07") + .withRow("patient-3", "2005-02-07") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 9 months", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 - 9 months", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("9 months", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000Z") - .withRow("patient-2", "2014-05-07T18:28:17.000Z") - .withRow("patient-3", "2014-05-07T18:28:17.000Z") + .withRow("patient-1", "2014-05-07") + .withRow("patient-2", "2014-05-07") + .withRow("patient-3", "2014-05-07") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 2 weeks", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 - 2 weeks", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000Z") - .withRow("patient-2", "2015-01-24T18:28:17.000Z") - .withRow("patient-3", "2015-01-24T18:28:17.000Z") + .withRow("patient-1", "2015-01-24") + .withRow("patient-2", "2015-01-24") + .withRow("patient-3", "2015-01-24") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 days", dateLiteralPath, + parameters.add(new TestParameters("@2015-02-07 - 30 days", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000Z") - .withRow("patient-2", "2015-01-08T18:28:17.000Z") - .withRow("patient-3", "2015-01-08T18:28:17.000Z") + .withRow("patient-1", "2015-01-08") + .withRow("patient-2", "2015-01-08") + .withRow("patient-3", "2015-01-08") .build()) ); - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 12 hours", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("12 hours", dateLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000Z") - .withRow("patient-2", "2015-02-07T06:28:17.000Z") - .withRow("patient-3", "2015-02-07T06:28:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 30 minutes", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000Z") - .withRow("patient-2", "2015-02-07T17:58:17.000Z") - .withRow("patient-3", "2015-02-07T17:58:17.000Z") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17.000Z - 10 seconds", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000Z") - .withRow("patient-2", "2015-02-07T18:28:07.000Z") - .withRow("patient-3", "2015-02-07T18:28:07.000Z") - .build()) - ); - - parameters.add( - new TestParameters("@2015-02-07T18:28:17.000Z - 300 milliseconds", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateLiteralPath), - context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700Z") - .withRow("patient-2", "2015-02-07T18:28:16.700Z") - .withRow("patient-3", "2015-02-07T18:28:16.700Z") - .build()) - ); return parameters; } diff --git a/fhir-server/src/test/java/au/csiro/pathling/sql/PruneSyntheticFieldsTest.java b/fhir-server/src/test/java/au/csiro/pathling/sql/PruneSyntheticFieldsTest.java index 11e67184da..6a884e6f2a 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/sql/PruneSyntheticFieldsTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/sql/PruneSyntheticFieldsTest.java @@ -54,7 +54,7 @@ public void testPruneSyntheticFields() { // Prune all columns. final Dataset prunedDataset = dataset.select( - Stream.of(dataset.columns()).map(dataset::col).map(PathlingFunctions::pruneSyntheticFields) + Stream.of(dataset.columns()).map(dataset::col).map(SqlExpressions::pruneSyntheticFields) .toArray(Column[]::new)); final Dataset expectedResult = new DatasetBuilder(spark) @@ -89,7 +89,7 @@ public void testPruneInGroupBy() { final Column valueColumn = dataset.col(dataset.columns()[dataset.columns().length - 1]); final Dataset groupedResult = dataset.groupBy( - PathlingFunctions.pruneSyntheticFields(valueColumn)) + SqlExpressions.pruneSyntheticFields(valueColumn)) .agg(functions.count(dataset.col("gender"))); DatasetAssert.of(groupedResult) diff --git a/fhir-server/src/test/java/au/csiro/pathling/sql/SqlExtensionsTest.java b/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java similarity index 96% rename from fhir-server/src/test/java/au/csiro/pathling/sql/SqlExtensionsTest.java rename to fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java index d6eae21130..a999643418 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/sql/SqlExtensionsTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java @@ -31,7 +31,7 @@ @SpringBootTest @Tag("UnitTest") -class SqlExtensionsTest { +class SqlOperationsTest { @Autowired SparkSession spark; @@ -81,9 +81,9 @@ void testMapWithPartitionPreview() { .withRow("patient-4", null, true) .build().repartition(1); - final Dataset resultDataset = SqlExtensions.mapWithPartitionPreview(dataset, + final Dataset resultDataset = SqlOperations.mapWithPartitionPreview(dataset, dataset.col("gender"), - SqlExtensionsTest::stringDecoder, + SqlOperationsTest::stringDecoder, new TestMapperWithPreview(), new StructField("myResult", DataTypes.IntegerType, true, Metadata.empty()) ); diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index ed0ff9cf41..b25a83a07d 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -12,12 +12,18 @@ import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; +import au.csiro.pathling.sql.dates.AddDurationToDate; +import au.csiro.pathling.sql.dates.AddDurationToDateTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromDate; +import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; +import au.csiro.pathling.sql.udf.SqlFunction2; import au.csiro.pathling.terminology.TerminologyService; -import au.csiro.pathling.terminology.ucum.ComparableQuantity; import au.csiro.pathling.terminology.ucum.Ucum; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; +import java.util.Collections; +import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; import org.apache.spark.sql.SparkSession; @@ -41,9 +47,12 @@ class UnitTestDependencies { @Nonnull static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, - @Nonnull final Optional sparkListener, - @Nonnull final ComparableQuantity comparableQuantity) { - return Spark.build(configuration, environment, sparkListener, comparableQuantity); + @Nonnull final Optional sparkListener) { + final List sqlFunction2 = List.of( + new AddDurationToDateTime(), new SubtractDurationFromDateTime(), new AddDurationToDate(), + new SubtractDurationFromDate()); + return Spark.build(configuration, environment, sparkListener, Collections.emptyList(), + sqlFunction2); } @Bean From b64bda34c3f6450f41da3bf4fe5cebd7bd822ec9 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 9 Apr 2022 14:57:45 +1000 Subject: [PATCH 10/83] Add support for arithmetic on Time --- .../pathling/fhirpath/element/TimePath.java | 18 ++- .../fhirpath/literal/TimeLiteralPath.java | 16 +- .../pathling/sql/dates/AddDurationToTime.java | 36 +++++ .../sql/dates/SubtractDurationFromTime.java | 36 +++++ .../sql/dates/TimeArithmeticFunction.java | 34 ++++ .../fhirpath/operator/DateArithmeticTest.java | 148 ++++++++++++++++++ .../pathling/test/UnitTestDependencies.java | 4 +- 7 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java index c2f4e25952..0dafe04c79 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java @@ -6,12 +6,19 @@ package au.csiro.pathling.fhirpath.element; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; + import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; +import au.csiro.pathling.sql.dates.AddDurationToTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromTime; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.Function; @@ -27,7 +34,8 @@ * * @author John Grimes */ -public class TimePath extends ElementPath implements Materializable, Comparable { +public class TimePath extends ElementPath implements Materializable, Comparable, + Temporal { private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(TimePath.class, TimeLiteralPath.class, NullLiteralPath.class); @@ -83,4 +91,12 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof TimeLiteralPath; } + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToTime.FUNCTION_NAME, SubtractDurationFromTime.FUNCTION_NAME); + } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java index 39f11651f9..d70a6743e3 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java @@ -6,12 +6,17 @@ package au.csiro.pathling.fhirpath.literal; +import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static au.csiro.pathling.utilities.Preconditions.check; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.Numeric.MathOperation; +import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.TimePath; +import au.csiro.pathling.sql.dates.AddDurationToTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromTime; import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; @@ -26,7 +31,8 @@ * * @author John Grimes */ -public class TimeLiteralPath extends LiteralPath implements Materializable, Comparable { +public class TimeLiteralPath extends LiteralPath implements Materializable, Comparable, + Temporal { @SuppressWarnings("WeakerAccess") protected TimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @@ -90,4 +96,12 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof TimePath; } + @Nonnull + @Override + public Function getDateArithmeticOperation( + @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, + @Nonnull final String expression) { + return buildDateArithmeticOperation(this, operation, dataset, expression, + AddDurationToTime.FUNCTION_NAME, SubtractDurationFromTime.FUNCTION_NAME); + } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java new file mode 100644 index 0000000000..349a44d809 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class AddDurationToTime extends TimeArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = 5839806160423512490L; + + public static final String FUNCTION_NAME = "add_duration_to_time"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performAddition; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java new file mode 100644 index 0000000000..7252dd4d53 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Quantity; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class SubtractDurationFromTime extends TimeArithmeticFunction implements + SqlFunction2 { + + private static final long serialVersionUID = 1909732257090337898L; + + public static final String FUNCTION_NAME = "subtract_duration_from_time"; + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + BiFunction getOperationFunction() { + return this::performSubtraction; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java new file mode 100644 index 0000000000..be1c3c7cbb --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import java.time.LocalTime; +import java.util.function.Function; +import java.util.regex.Pattern; +import org.hl7.fhir.r4.model.TimeType; + +public abstract class TimeArithmeticFunction extends TemporalArithmeticFunction { + + private static final long serialVersionUID = 7504107794184508523L; + + private static final Pattern HOURS_ONLY = Pattern.compile("^\\d{2}$"); + + @Override + Function parseEncodedValue() { + // LocalDate will not successfully parse the HH format. + return (temporal) -> HOURS_ONLY.matcher(temporal).matches() + ? LocalTime.parse(temporal + ":00") + : LocalTime.parse(temporal); + } + + @Override + Function encodeResult() { + return (resultTime) -> new TimeType(resultTime.toString()) + .getValueAsString(); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index cad821700d..3f8b906ea4 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -13,6 +13,7 @@ import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.test.builders.DatasetBuilder; import au.csiro.pathling.test.builders.ElementPathBuilder; @@ -112,9 +113,24 @@ Stream parameters() throws ParseException { .singular(true) .build(); + final Dataset timeDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "13:28:17") + .withRow("patient-2", "08:00") + .withRow("patient-3", "00") + .build(); + final ElementPath timePath = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.TIME) + .dataset(timeDataset) + .idAndValueColumns() + .singular(true) + .build(); + final DateTimeLiteralPath dateTimeLiteral = DateTimeLiteralPath.fromString( "@2015-02-07T18:28:17+00:00", dateTimePath); final DateLiteralPath dateLiteral = DateLiteralPath.fromString("@2015-02-07", datePath); + final TimeLiteralPath timeLiteral = TimeLiteralPath.fromString("@T08:00", timePath); final ParserContext context = new ParserContextBuilder(spark, fhirContext) .groupingColumns(Collections.singletonList(dateTimePath.getIdColumn())) @@ -124,10 +140,14 @@ Stream parameters() throws ParseException { parameters.addAll(dateTimeSubtraction(dateTimePath, context)); parameters.addAll(dateAddition(datePath, context)); parameters.addAll(dateSubtraction(datePath, context)); + parameters.addAll(timeAddition(timePath, context)); + parameters.addAll(timeSubtraction(timePath, context)); parameters.addAll(dateTimeLiteralAddition(dateTimeLiteral, context)); parameters.addAll(dateTimeLiteralSubtraction(dateTimeLiteral, context)); parameters.addAll(dateLiteralAddition(dateLiteral, context)); parameters.addAll(dateLiteralSubtraction(dateLiteral, context)); + parameters.addAll(timeLiteralAddition(timeLiteral, context)); + parameters.addAll(timeLiteralSubtraction(timeLiteral, context)); return parameters.stream(); } @@ -628,6 +648,134 @@ Collection dateLiteralSubtraction( return parameters; } + Collection timeAddition(final FhirPath timePath, + final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("Time + 6 hours", timePath, + QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "19:28:17") + .withRow("patient-2", "14:00") + .withRow("patient-3", "06:00") + .build())); + + parameters.add(new TestParameters("Time + 45 minutes", timePath, + QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "14:13:17") + .withRow("patient-2", "08:45") + .withRow("patient-3", "00:45") + .build())); + + parameters.add(new TestParameters("Time + 15 seconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "13:28:32") + .withRow("patient-2", "08:00:15") + .withRow("patient-3", "00:00:15") + .build())); + return parameters; + } + + Collection timeSubtraction(final FhirPath timePath, + final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("Time - 6 hours", timePath, + QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "07:28:17") + .withRow("patient-2", "02:00") + .withRow("patient-3", "18:00") + .build())); + + parameters.add(new TestParameters("Time - 45 minutes", timePath, + QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "12:43:17") + .withRow("patient-2", "07:15") + .withRow("patient-3", "23:15") + .build())); + + parameters.add(new TestParameters("Time - 15 seconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "13:28:02") + .withRow("patient-2", "07:59:45") + .withRow("patient-3", "23:59:45") + .build())); + return parameters; + } + + Collection timeLiteralAddition(final FhirPath timePath, + final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@T08:00 + 6 hours", timePath, + QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "14:00") + .withRow("patient-2", "14:00") + .withRow("patient-3", "14:00") + .build())); + + parameters.add(new TestParameters("@T08:00 + 45 minutes", timePath, + QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "08:45") + .withRow("patient-2", "08:45") + .withRow("patient-3", "08:45") + .build())); + + parameters.add(new TestParameters("@T08:00 + 15 seconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "08:00:15") + .withRow("patient-2", "08:00:15") + .withRow("patient-3", "08:00:15") + .build())); + return parameters; + } + + Collection timeLiteralSubtraction(final FhirPath timePath, + final ParserContext context) { + final List parameters = new ArrayList<>(); + parameters.add(new TestParameters("@T08:00 - 6 hours", timePath, + QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "02:00") + .withRow("patient-2", "02:00") + .withRow("patient-3", "02:00") + .build())); + + parameters.add(new TestParameters("@T08:00 - 45 minutes", timePath, + QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "07:15") + .withRow("patient-2", "07:15") + .withRow("patient-3", "07:15") + .build())); + + parameters.add(new TestParameters("@T08:00 - 15 seconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "07:59:45") + .withRow("patient-2", "07:59:45") + .withRow("patient-3", "07:59:45") + .build())); + return parameters; + } + @ParameterizedTest @MethodSource("parameters") void test(@Nonnull final TestParameters parameters) { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index b25a83a07d..3b34f73470 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -14,8 +14,10 @@ import au.csiro.pathling.spark.Spark; import au.csiro.pathling.sql.dates.AddDurationToDate; import au.csiro.pathling.sql.dates.AddDurationToDateTime; +import au.csiro.pathling.sql.dates.AddDurationToTime; import au.csiro.pathling.sql.dates.SubtractDurationFromDate; import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; +import au.csiro.pathling.sql.dates.SubtractDurationFromTime; import au.csiro.pathling.sql.udf.SqlFunction2; import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.terminology.ucum.Ucum; @@ -50,7 +52,7 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Optional sparkListener) { final List sqlFunction2 = List.of( new AddDurationToDateTime(), new SubtractDurationFromDateTime(), new AddDurationToDate(), - new SubtractDurationFromDate()); + new SubtractDurationFromDate(), new AddDurationToTime(), new SubtractDurationFromTime()); return Spark.build(configuration, environment, sparkListener, Collections.emptyList(), sqlFunction2); } From c44e1b95acb516dc62f6de369c6bc4ca2b10b4ab Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 9 Apr 2022 15:04:28 +1000 Subject: [PATCH 11/83] Remove unnecessary implements clauses --- .../java/au/csiro/pathling/sql/dates/AddDurationToDate.java | 5 +---- .../au/csiro/pathling/sql/dates/AddDurationToDateTime.java | 5 +---- .../java/au/csiro/pathling/sql/dates/AddDurationToTime.java | 5 +---- .../csiro/pathling/sql/dates/SubtractDurationFromDate.java | 5 +---- .../csiro/pathling/sql/dates/SubtractDurationFromTime.java | 5 +---- 5 files changed, 5 insertions(+), 20 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java index 3dfa42712e..7e4a54304e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java @@ -6,18 +6,15 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.sql.udf.SqlFunction2; import java.time.LocalDate; import java.util.function.BiFunction; -import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component @Profile("core") -public class AddDurationToDate extends DateArithmeticFunction implements - SqlFunction2 { +public class AddDurationToDate extends DateArithmeticFunction { private static final long serialVersionUID = -5029179160644275584L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java index 48c473458f..3f09e1d3bd 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java @@ -6,18 +6,15 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.sql.udf.SqlFunction2; import java.time.ZonedDateTime; import java.util.function.BiFunction; -import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component @Profile("core") -public class AddDurationToDateTime extends DateTimeArithmeticFunction implements - SqlFunction2 { +public class AddDurationToDateTime extends DateTimeArithmeticFunction { private static final long serialVersionUID = 6922227603585641053L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java index 349a44d809..e889c2ee80 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java @@ -6,18 +6,15 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.sql.udf.SqlFunction2; import java.time.LocalTime; import java.util.function.BiFunction; -import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component @Profile("core") -public class AddDurationToTime extends TimeArithmeticFunction implements - SqlFunction2 { +public class AddDurationToTime extends TimeArithmeticFunction { private static final long serialVersionUID = 5839806160423512490L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java index 7566472591..65ad41b160 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java @@ -6,18 +6,15 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.sql.udf.SqlFunction2; import java.time.LocalDate; import java.util.function.BiFunction; -import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component @Profile("core") -public class SubtractDurationFromDate extends DateArithmeticFunction implements - SqlFunction2 { +public class SubtractDurationFromDate extends DateArithmeticFunction { private static final long serialVersionUID = 5201879133976866457L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java index 7252dd4d53..7dcaa51f98 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java @@ -6,18 +6,15 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.sql.udf.SqlFunction2; import java.time.LocalTime; import java.util.function.BiFunction; -import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @Component @Profile("core") -public class SubtractDurationFromTime extends TimeArithmeticFunction implements - SqlFunction2 { +public class SubtractDurationFromTime extends TimeArithmeticFunction { private static final long serialVersionUID = 1909732257090337898L; From 9438b82ff125fe6c876c9ad12aa49ee6074fc302 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 9 Apr 2022 15:09:09 +1000 Subject: [PATCH 12/83] Add ComparableQuantity to unit test dependencies --- .../pathling/test/UnitTestDependencies.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 3b34f73470..126100d5ef 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -18,13 +18,14 @@ import au.csiro.pathling.sql.dates.SubtractDurationFromDate; import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; import au.csiro.pathling.sql.dates.SubtractDurationFromTime; +import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; import au.csiro.pathling.terminology.TerminologyService; +import au.csiro.pathling.terminology.ucum.ComparableQuantity; import au.csiro.pathling.terminology.ucum.Ucum; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; -import java.util.Collections; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; @@ -49,12 +50,13 @@ class UnitTestDependencies { @Nonnull static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, - @Nonnull final Optional sparkListener) { - final List sqlFunction2 = List.of( - new AddDurationToDateTime(), new SubtractDurationFromDateTime(), new AddDurationToDate(), - new SubtractDurationFromDate(), new AddDurationToTime(), new SubtractDurationFromTime()); - return Spark.build(configuration, environment, sparkListener, Collections.emptyList(), - sqlFunction2); + @Nonnull final Optional sparkListener, + @Nonnull final UcumService ucumService) { + final List sqlFunction1 = List.of(new ComparableQuantity(ucumService)); + final List sqlFunction2 = List.of(new AddDurationToDateTime(), + new SubtractDurationFromDateTime(), new AddDurationToDate(), new SubtractDurationFromDate(), + new AddDurationToTime(), new SubtractDurationFromTime()); + return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2); } @Bean From 70202deacfe533bcacabafe5d90a0f6c7dc4e059 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 9 Apr 2022 15:27:08 +1000 Subject: [PATCH 13/83] Remove support for milliseconds and document --- .../fhirpath/parser/generated/FhirPath.g4 | 4 +-- .../fhirpath/element/DateTimePath.java | 12 +++------ .../sql/dates/TemporalArithmeticFunction.java | 2 -- .../terminology/ucum/ComparableQuantity.java | 2 -- site/docs/fhirpath/data-types.md | 13 +++------- site/docs/fhirpath/operators.md | 26 ++++++++++--------- 6 files changed, 24 insertions(+), 35 deletions(-) diff --git a/fhir-server/src/main/antlr4/au/csiro/pathling/fhirpath/parser/generated/FhirPath.g4 b/fhir-server/src/main/antlr4/au/csiro/pathling/fhirpath/parser/generated/FhirPath.g4 index c4a7c3f74e..4a82cd1e17 100644 --- a/fhir-server/src/main/antlr4/au/csiro/pathling/fhirpath/parser/generated/FhirPath.g4 +++ b/fhir-server/src/main/antlr4/au/csiro/pathling/fhirpath/parser/generated/FhirPath.g4 @@ -80,11 +80,11 @@ unit ; dateTimePrecision - : 'year' | 'month' | 'week' | 'day' | 'hour' | 'minute' | 'second' | 'millisecond' + : 'year' | 'month' | 'week' | 'day' | 'hour' | 'minute' | 'second' ; pluralDateTimePrecision - : 'years' | 'months' | 'weeks' | 'days' | 'hours' | 'minutes' | 'seconds' | 'milliseconds' + : 'years' | 'months' | 'weeks' | 'days' | 'hours' | 'minutes' | 'seconds' ; typeSpecifier diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index afe1000e4e..6a906ee236 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -46,16 +46,12 @@ public class DateTimePath extends ElementPath implements Materializable FULL_DATE_FORMAT = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private static final ThreadLocal SECONDS_DATE_FORMAT = dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); private static final ThreadLocal MINUTES_DATE_FORMAT = dateFormatThreadLocal("yyyy-MM-dd'T'HH:mmXXX"); private static final ThreadLocal HOURS_DATE_FORMAT = dateFormatThreadLocal("yyyy-MM-dd'T'HHXXX"); - private static final ThreadLocal FULL_DATE_FORMAT_NO_TZ = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private static final ThreadLocal SECONDS_DATE_FORMAT_NO_TZ = dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); private static final ThreadLocal MINUTES_DATE_FORMAT_NO_TZ = @@ -127,13 +123,13 @@ public static Function buildComparison(@Nonnull final Compar } public static SimpleDateFormat getDefaultDateFormat() { - return FULL_DATE_FORMAT.get(); + return SECONDS_DATE_FORMAT.get(); } public static List getAllDateFormats() { - return List.of(FULL_DATE_FORMAT.get(), SECONDS_DATE_FORMAT.get(), MINUTES_DATE_FORMAT.get(), - HOURS_DATE_FORMAT.get(), FULL_DATE_FORMAT_NO_TZ.get(), SECONDS_DATE_FORMAT_NO_TZ.get(), - MINUTES_DATE_FORMAT_NO_TZ.get(), HOURS_DATE_FORMAT_NO_TZ.get()); + return List.of(SECONDS_DATE_FORMAT.get(), MINUTES_DATE_FORMAT.get(), HOURS_DATE_FORMAT.get(), + SECONDS_DATE_FORMAT_NO_TZ.get(), MINUTES_DATE_FORMAT_NO_TZ.get(), + HOURS_DATE_FORMAT_NO_TZ.get()); } public static TimeZone getTimeZone() { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index db701415b6..ca12b8707b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -44,8 +44,6 @@ public abstract class TemporalArithmeticFunction { private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() .put("second", "s") .put("seconds", "s") - .put("millisecond", "ms") - .put("milliseconds", "ms") .build(); /** diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index f72f445867..d4a015aebd 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -112,17 +112,13 @@ The Time literal uses a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601): - A time begins with a `@T` -- It uses the `Thh:mm:ss.ffff±hh:mm` format, though minute, second, millisecond - parts are optional -- Timezone is optional, but if present the notation `±hh:mm` is used (so must - include both minutes and hours) -- `Z` is allowed as a synonym for the zero (+00:00) UTC offset. +- It uses the `Thh:mm:ss` format, though minute and second are optional +- Milliseconds are not supported Examples: ``` -@T07:30:14.559-07:00 -@T14:30:14.559Z +@T07:30:14 @T14:30 @T14 ``` @@ -131,7 +127,7 @@ Examples: The DateTime literal combines the [Date](#date) and [Time](#time) literals and is a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). It uses the -`YYYY-MM-DDThh:mm:ss.ffff±hh:mm` format. +`YYYY-MM-DDThh:mm:ss±hh:mm` format. Milliseconds are not supported. Example: @@ -161,7 +157,6 @@ The calendar duration keywords that are supported are: - `hour` / `hours` - `minute` / `minutes` - `second` / `seconds` -- `millisecond` / `milliseconds` Example: diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index 94d19c0fc3..02f0c9e8a2 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -112,18 +112,20 @@ See also: [Math](https://hl7.org/fhirpath/#math) The following operators are supported for date arithmetic: -- `+` - Add a duration to a [Date](./data-types.html#date) - or [DateTime](./data-types.html#datetime) -- `-` - Subtract a duration from a [Date](./data-types.html#date) - or [DateTime](./data-types.html#datetime) - -The duration operand is a [Quantity](./data-types.html#quantity) literal, and -must be a [calendar duration](https://hl7.org/fhirpath/#time-valued-quantities) -as defined in the FHIRPath specification. The use of UCUM units is not supported -with these operators. - -The Date or DateTime operand must be singular. If it is an empty collection, the -operator will return an empty collection. +- `+` - Add a duration to a [Date](./data-types.html#date), + [DateTime](./data-types.html#datetime) or [Time](./data-types.html#time) +- `-` - Subtract a duration from a [Date](./data-types.html#date), + [DateTime](./data-types.html#datetime) or [Time](./data-types.html#time) + +The duration operand is a +[calendar duration literal](./data-types.html#quantity). The use of UCUM units +is not supported with these operators. + +The `Date` or `DateTime` operand must be singular. If it is an empty collection, +the operator will return an empty collection. + +The `Date` type does not support arithmetic with durations below `day/days`. The +`Time` type does not support arithmetic with durations above `hour/hours`. See also: [Date/Time Arithmetic](https://hl7.org/fhirpath/#datetime-arithmetic) From 160b0033daf41bea8aaf2a6bc79bb5cf07c78fe1 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Apr 2022 16:38:37 +1000 Subject: [PATCH 14/83] Get all tests passing, refactor some aspects of literals --- .../pathling/fhirpath/element/DatePath.java | 6 +- .../fhirpath/element/DateTimePath.java | 44 +--- .../function/memberof/MemberOfFunction.java | 2 +- .../function/translate/TranslateFunction.java | 25 +- .../fhirpath/literal/BooleanLiteralPath.java | 21 +- .../fhirpath/literal/CodingLiteralPath.java | 35 +-- .../fhirpath/literal/DateLiteralPath.java | 69 +---- .../fhirpath/literal/DateTimeLiteralPath.java | 57 ++-- .../fhirpath/literal/DecimalLiteralPath.java | 23 +- .../fhirpath/literal/IntegerLiteralPath.java | 22 +- .../fhirpath/literal/LiteralPath.java | 52 ++-- .../fhirpath/literal/NullLiteralPath.java | 10 +- .../fhirpath/literal/QuantityLiteralPath.java | 43 ++- .../fhirpath/literal/StringLiteralPath.java | 22 +- .../fhirpath/literal/TimeLiteralPath.java | 29 +- .../operator/DateArithmeticOperator.java | 2 +- .../sql/dates/DateArithmeticFunction.java | 2 - .../sql/dates/DateTimeArithmeticFunction.java | 11 +- .../sql/dates/TemporalArithmeticFunction.java | 5 + .../terminology/ucum/ComparableQuantity.java | 2 + .../literal/CodingLiteralPathTest.java | 12 +- .../fhirpath/operator/DateArithmeticTest.java | 248 ++++++++++++------ .../pathling/fhirpath/parser/ParserTest.java | 67 ++++- .../pathling/test/UnitTestDependencies.java | 4 +- .../test/assertions/LiteralPathAssertion.java | 9 +- .../queryWithDateTimeGrouping.Parameters.json | 86 +++--- .../DateTimeLiteral.Parameters.json | 4 +- site/docs/fhirpath/data-types.md | 34 ++- site/docs/fhirpath/operators.md | 11 +- 29 files changed, 481 insertions(+), 476 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java index b25028b6da..536b6629b2 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java @@ -45,19 +45,19 @@ public class DatePath extends ElementPath implements Materializable, C private static final ThreadLocal FULL_DATE_FORMAT = ThreadLocal .withInitial(() -> { final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.setTimeZone(DateTimePath.getTimeZone()); + format.setTimeZone(DateTimePath.getDefaultTimeZone()); return format; }); private static final ThreadLocal YEAR_MONTH_DATE_FORMAT = ThreadLocal .withInitial(() -> { final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM"); - format.setTimeZone(DateTimePath.getTimeZone()); + format.setTimeZone(DateTimePath.getDefaultTimeZone()); return format; }); private static final ThreadLocal YEAR_ONLY_DATE_FORMAT = ThreadLocal .withInitial(() -> { final SimpleDateFormat format = new SimpleDateFormat("yyyy"); - format.setTimeZone(DateTimePath.getTimeZone()); + format.setTimeZone(DateTimePath.getDefaultTimeZone()); return format; }); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index 6a906ee236..789c3a496c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -22,8 +22,6 @@ import au.csiro.pathling.sql.dates.AddDurationToDateTime; import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; import com.google.common.collect.ImmutableSet; -import java.text.SimpleDateFormat; -import java.util.List; import java.util.Optional; import java.util.TimeZone; import java.util.function.BiFunction; @@ -45,19 +43,7 @@ public class DateTimePath extends ElementPath implements Materializable, Comparable, Temporal { - private static final TimeZone TIME_ZONE = TimeZone.getTimeZone("GMT"); - private static final ThreadLocal SECONDS_DATE_FORMAT = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); - private static final ThreadLocal MINUTES_DATE_FORMAT = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mmXXX"); - private static final ThreadLocal HOURS_DATE_FORMAT = - dateFormatThreadLocal("yyyy-MM-dd'T'HHXXX"); - private static final ThreadLocal SECONDS_DATE_FORMAT_NO_TZ = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mm:ssXXX"); - private static final ThreadLocal MINUTES_DATE_FORMAT_NO_TZ = - dateFormatThreadLocal("yyyy-MM-dd'T'HH:mmXXX"); - private static final ThreadLocal HOURS_DATE_FORMAT_NO_TZ = - dateFormatThreadLocal("yyyy-MM-dd'T'HHXXX"); + private static final TimeZone DEFAULT_TIME_ZONE = TimeZone.getTimeZone("GMT"); private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(DatePath.class, DateTimePath.class, DateLiteralPath.class, DateTimeLiteralPath.class, @@ -96,11 +82,11 @@ public static Optional valueFromRow(@Nonnull final Row row, if (fhirType == FHIRDefinedType.INSTANT) { final InstantType value = new InstantType(row.getTimestamp(columnNumber)); - value.setTimeZone(TIME_ZONE); + value.setTimeZone(DEFAULT_TIME_ZONE); return Optional.of(value); } else { final DateTimeType value = new DateTimeType(row.getString(columnNumber)); - value.setTimeZone(TIME_ZONE); + value.setTimeZone(DEFAULT_TIME_ZONE); return Optional.of(value); } } @@ -122,18 +108,8 @@ public static Function buildComparison(@Nonnull final Compar .apply(to_timestamp(source.getValueColumn()), to_timestamp(target.getValueColumn())); } - public static SimpleDateFormat getDefaultDateFormat() { - return SECONDS_DATE_FORMAT.get(); - } - - public static List getAllDateFormats() { - return List.of(SECONDS_DATE_FORMAT.get(), MINUTES_DATE_FORMAT.get(), HOURS_DATE_FORMAT.get(), - SECONDS_DATE_FORMAT_NO_TZ.get(), MINUTES_DATE_FORMAT_NO_TZ.get(), - HOURS_DATE_FORMAT_NO_TZ.get()); - } - - public static TimeZone getTimeZone() { - return TIME_ZONE; + public static TimeZone getDefaultTimeZone() { + return DEFAULT_TIME_ZONE; } @Nonnull @@ -157,16 +133,6 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof DateTimeLiteralPath; } - @Nonnull - private static ThreadLocal dateFormatThreadLocal(final String pattern) { - return ThreadLocal - .withInitial(() -> { - final SimpleDateFormat format = new SimpleDateFormat(pattern); - format.setTimeZone(TIME_ZONE); - return format; - }); - } - @Nonnull @Override public Function getDateArithmeticOperation( diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java index 383db572d1..403abd814b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/memberof/MemberOfFunction.java @@ -80,7 +80,7 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { @SuppressWarnings("OptionalGetWithoutIsPresent") final TerminologyServiceFactory terminologyServiceFactory = inputContext .getTerminologyServiceFactory().get(); - final String valueSetUri = argument.getJavaValue(); + final String valueSetUri = argument.getValue().getValueAsString(); final Dataset dataset = inputPath.getDataset(); // Perform a validate code operation on each Coding or CodeableConcept in the input dataset, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java index 9f1be3bcab..be2b0efc51 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/translate/TranslateFunction.java @@ -43,7 +43,10 @@ import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.StructField; +import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.Enumerations.ConceptMapEquivalence; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Type; import org.slf4j.MDC; /** @@ -94,7 +97,7 @@ private Arguments(@Nonnull final List arguments) { */ @SuppressWarnings("unchecked") @Nonnull - private T getValueOr(final int index, @Nonnull final T defaultValue) { + private T getValueOr(final int index, @Nonnull final T defaultValue) { return (index < arguments.size()) ? getValue(index, (Class) defaultValue.getClass()) : defaultValue; @@ -103,15 +106,15 @@ private T getValueOr(final int index, @Nonnull final T defaultValue) { /** * Gets the value of the required literal argument. * - * @param index the 0-based index of the argument. - * @param valueClass the expected Java class of the argument value. - * @param the Java type of the argument value. - * @return the java value of the requested argument. + * @param index the 0-based index of the argument + * @param valueClass the expected Java class of the argument value + * @param the HAPI type of the argument value + * @return the java value of the requested argument */ @Nonnull - public T getValue(final int index, @Nonnull final Class valueClass) { + public T getValue(final int index, @Nonnull final Class valueClass) { return Objects - .requireNonNull(valueClass.cast(((LiteralPath) arguments.get(index)).getJavaValue())); + .requireNonNull(valueClass.cast(((LiteralPath) arguments.get(index)).getValue())); } /** @@ -158,9 +161,11 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { final Arguments arguments = Arguments.of(input); - final String conceptMapUrl = arguments.getValue(0, String.class); - final boolean reverse = arguments.getValueOr(1, DEFAULT_REVERSE); - final String equivalence = arguments.getValueOr(2, DEFAULT_EQUIVALENCE); + final String conceptMapUrl = arguments.getValue(0, StringType.class).asStringValue(); + final boolean reverse = arguments.getValueOr(1, new BooleanType(DEFAULT_REVERSE)) + .booleanValue(); + final String equivalence = arguments.getValueOr(2, new StringType(DEFAULT_EQUIVALENCE)) + .asStringValue(); final Dataset dataset = inputPath.getDataset(); final MapperWithPreview, Row[], ConceptTranslator> mapper = diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java index 853b98289a..4e8a62edca 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java @@ -6,7 +6,7 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; +import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -19,21 +19,19 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.BooleanType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath boolean literal. * * @author John Grimes */ -public class BooleanLiteralPath extends LiteralPath implements Materializable, - Comparable { +public class BooleanLiteralPath extends LiteralPath implements + Materializable, Comparable { @SuppressWarnings("WeakerAccess") protected BooleanLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final BooleanType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof BooleanType); } /** @@ -55,18 +53,13 @@ public static BooleanLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull @Override public String getExpression() { - return getLiteralValue().asStringValue(); - } - - @Override - public BooleanType getLiteralValue() { - return (BooleanType) literalValue; + return getValue().asStringValue(); } @Nonnull @Override - public Boolean getJavaValue() { - return getLiteralValue().booleanValue(); + public Column buildValueColumn() { + return lit(getValue().booleanValue()); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java index 97fac6b574..beddfd7096 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java @@ -6,7 +6,6 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.struct; @@ -22,7 +21,6 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Coding; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath Coding literal. @@ -30,13 +28,17 @@ * @author John Grimes */ @Getter -public class CodingLiteralPath extends LiteralPath implements Materializable, Comparable { +public class CodingLiteralPath extends LiteralPath implements Materializable, + Comparable { - @SuppressWarnings("WeakerAccess") protected CodingLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final Coding literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof Coding); + } + + protected CodingLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final Coding literalValue, @Nonnull final String expression) { + super(dataset, idColumn, literalValue, expression); } /** @@ -49,34 +51,23 @@ protected CodingLiteralPath(@Nonnull final Dataset dataset, @Nonnull final * @throws IllegalArgumentException if the literal is malformed */ @Nonnull - public static CodingLiteralPath fromString(@Nonnull final CharSequence fhirPath, + public static CodingLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) throws IllegalArgumentException { return new CodingLiteralPath(context.getDataset(), context.getIdColumn(), - CodingLiteral.fromString(fhirPath)); + CodingLiteral.fromString(fhirPath), fhirPath); } @Nonnull @Override public String getExpression() { - return CodingLiteral.toLiteral(getLiteralValue()); - - } + return expression.orElse(CodingLiteral.toLiteral(getValue())); - @Override - public Coding getLiteralValue() { - return (Coding) literalValue; - } - - @Nonnull - @Override - public Coding getJavaValue() { - return getLiteralValue(); } @Nonnull @Override public Column buildValueColumn() { - final Coding value = getJavaValue(); + final Coding value = getValue(); return struct( lit(value.getId()).as("id"), lit(value.getSystem()).as("system"), @@ -114,7 +105,7 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { @Nonnull @Override public Column getExtractableColumn() { - return lit(CodingLiteral.toLiteral(getLiteralValue())); + return lit(CodingLiteral.toLiteral(getValue())); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java index 90068295ab..3584aafc88 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java @@ -7,7 +7,6 @@ package au.csiro.pathling.fhirpath.literal; import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; -import static au.csiro.pathling.utilities.Preconditions.check; import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; @@ -20,7 +19,6 @@ import au.csiro.pathling.sql.dates.AddDurationToDate; import au.csiro.pathling.sql.dates.SubtractDurationFromDate; import java.text.ParseException; -import java.util.Date; import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; @@ -28,25 +26,23 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.DateType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath date literal. * * @author John Grimes */ -public class DateLiteralPath extends LiteralPath implements Materializable, Comparable, - Temporal { +public class DateLiteralPath extends LiteralPath implements Materializable, + Comparable, Temporal { - @Nonnull - private Optional format; - - @SuppressWarnings("WeakerAccess") protected DateLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final DateType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof DateType); - format = Optional.empty(); + } + + protected DateLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final DateType literalValue, @Nonnull final String expression) { + super(dataset, idColumn, literalValue, expression); } /** @@ -61,57 +57,20 @@ protected DateLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Co public static DateLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) throws ParseException { final String dateString = fhirPath.replaceFirst("^@", ""); - java.util.Date date; - DateLiteralFormat format; - // Try parsing out the date using the three possible formats, from full (most common) down to - // the year only format. - try { - date = DatePath.getFullDateFormat().parse(dateString); - format = DateLiteralFormat.FULL; - } catch (final ParseException e) { - try { - date = DatePath.getYearMonthDateFormat().parse(dateString); - format = DateLiteralFormat.YEAR_MONTH_DATE; - } catch (final ParseException ex) { - date = DatePath.getYearOnlyDateFormat().parse(dateString); - format = DateLiteralFormat.YEAR_ONLY; - } - } - - final DateLiteralPath result = new DateLiteralPath(context.getDataset(), context.getIdColumn(), - new DateType(date)); - result.format = Optional.of(format); - return result; + final DateType dateType = new DateType(dateString); + return new DateLiteralPath(context.getDataset(), context.getIdColumn(), dateType, fhirPath); } @Nonnull @Override public String getExpression() { - if (format.isEmpty() || format.get() == DateLiteralFormat.FULL) { - return "@" + DatePath.getFullDateFormat().format(getLiteralValue().getValue()); - } else if (format.get() == DateLiteralFormat.YEAR_MONTH_DATE) { - return "@" + DatePath.getYearMonthDateFormat().format(getLiteralValue().getValue()); - } else { - return "@" + DatePath.getYearOnlyDateFormat().format(getLiteralValue().getValue()); - } - } - - @Override - public DateType getLiteralValue() { - return (DateType) literalValue; - } - - @Nonnull - @Override - public Date getJavaValue() { - return getLiteralValue().getValue(); + return expression.orElse("@" + getValue().asStringValue()); } @Nonnull @Override public Column buildValueColumn() { - final String date = DatePath.getFullDateFormat().format(getJavaValue()); - return lit(date); + return lit(getValue().asStringValue()); } @Override @@ -145,8 +104,4 @@ public Function getDateArithmeticOperation( AddDurationToDate.FUNCTION_NAME, SubtractDurationFromDate.FUNCTION_NAME); } - private enum DateLiteralFormat { - FULL, YEAR_MONTH_DATE, YEAR_ONLY - } - } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java index 707fa50830..ba1b3bb3f8 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java @@ -7,8 +7,6 @@ package au.csiro.pathling.fhirpath.literal; import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; -import static au.csiro.pathling.utilities.Preconditions.check; -import static au.csiro.pathling.utilities.Preconditions.checkNotNull; import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; @@ -20,8 +18,6 @@ import au.csiro.pathling.sql.dates.AddDurationToDateTime; import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; @@ -31,21 +27,23 @@ import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath date literal. * * @author John Grimes */ -public class DateTimeLiteralPath extends LiteralPath implements Materializable, - Comparable, Temporal { +public class DateTimeLiteralPath extends LiteralPath implements + Materializable, Comparable, Temporal { - @SuppressWarnings("WeakerAccess") protected DateTimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final DateTimeType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof DateTimeType); + } + + protected DateTimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final DateTimeType literalValue, @Nonnull final String expression) { + super(dataset, idColumn, literalValue, expression); } /** @@ -57,51 +55,25 @@ protected DateTimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull fina * @return A new instance of {@link LiteralPath} * @throws ParseException if the literal is malformed */ + @Nonnull public static DateTimeLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) throws ParseException { final String dateTimeString = fhirPath.replaceFirst("^@", ""); - final java.util.Date date = parseDateTimeString(dateTimeString); - final DateTimeType literalValue = new DateTimeType(date); - literalValue.setTimeZone(DateTimePath.getTimeZone()); - return new DateTimeLiteralPath(context.getDataset(), context.getIdColumn(), literalValue); - } - - @Nonnull - private static Date parseDateTimeString(@Nonnull final String dateTimeString) - throws ParseException { - ParseException parseException = null; - for (final SimpleDateFormat format : DateTimePath.getAllDateFormats()) { - try { - return format.parse(dateTimeString); - } catch (final ParseException e) { - parseException = e; - } - } - checkNotNull(parseException); - throw parseException; + final DateTimeType dateTimeType = new DateTimeType(dateTimeString); + return new DateTimeLiteralPath(context.getDataset(), context.getIdColumn(), dateTimeType, + fhirPath); } @Nonnull @Override public String getExpression() { - return "@" + DateTimePath.getDefaultDateFormat().format(getLiteralValue().getValue()); - } - - @Override - public DateTimeType getLiteralValue() { - return (DateTimeType) literalValue; - } - - @Nonnull - @Override - public Date getJavaValue() { - return getLiteralValue().getValue(); + return expression.orElse("@" + getValue().getValueAsString()); } @Nonnull @Override public Column buildValueColumn() { - return lit(getLiteralValue().asStringValue()); + return lit(getValue().asStringValue()); } @Override @@ -135,4 +107,5 @@ public Function getDateArithmeticOperation( return buildDateArithmeticOperation(this, operation, dataset, expression, AddDurationToDateTime.FUNCTION_NAME, SubtractDurationFromDateTime.FUNCTION_NAME); } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java index a2ad52184a..b7244d55d8 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java @@ -6,7 +6,7 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; +import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; @@ -25,21 +25,18 @@ import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.DecimalType; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath decimal literal. * * @author John Grimes */ -public class DecimalLiteralPath extends LiteralPath implements Materializable, - Comparable, Numeric { +public class DecimalLiteralPath extends LiteralPath implements + Materializable, Comparable, Numeric { - @SuppressWarnings("WeakerAccess") protected DecimalLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final DecimalType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof DecimalType); } /** @@ -73,19 +70,13 @@ public static DecimalLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull @Override public String getExpression() { - return getLiteralValue().getValue().toPlainString(); - } - - @Override - @Nonnull - public DecimalType getLiteralValue() { - return (DecimalType) literalValue; + return getValue().getValue().toPlainString(); } @Nonnull @Override - public BigDecimal getJavaValue() { - return getLiteralValue().getValue(); + public Column buildValueColumn() { + return lit(getValue().getValue()); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java index bdf8c5f905..730519b396 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java @@ -6,7 +6,7 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; +import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -24,21 +24,19 @@ import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.IntegerType; import org.hl7.fhir.r4.model.PrimitiveType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath integer literal. * * @author John Grimes */ -public class IntegerLiteralPath extends LiteralPath implements Materializable, - Comparable, Numeric { +public class IntegerLiteralPath extends LiteralPath implements + Materializable, Comparable, Numeric { @SuppressWarnings("WeakerAccess") protected IntegerLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final PrimitiveType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof IntegerType); } /** @@ -60,19 +58,13 @@ public static IntegerLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull @Override public String getExpression() { - return getLiteralValue().getValueAsString(); - } - - @Override - @Nonnull - public IntegerType getLiteralValue() { - return (IntegerType) literalValue; + return getValue().getValueAsString(); } @Nonnull @Override - public Integer getJavaValue() { - return getLiteralValue().getValue(); + public Column buildValueColumn() { + return lit(getValue().getValue()); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java index 850d4d1124..b750dcc9ed 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java @@ -7,7 +7,6 @@ package au.csiro.pathling.fhirpath.literal; import static au.csiro.pathling.QueryHelpers.getUnionableColumns; -import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.FhirPath; @@ -16,10 +15,11 @@ import com.google.common.collect.ImmutableMap; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import javolution.testing.AssertionException; import lombok.Getter; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; @@ -32,7 +32,7 @@ * * @author John Grimes */ -public abstract class LiteralPath implements FhirPath { +public abstract class LiteralPath implements FhirPath { // See https://hl7.org/fhir/fhirpath.html#types. private static final Map> FHIR_TYPE_TO_FHIRPATH_TYPE = @@ -87,14 +87,27 @@ public abstract class LiteralPath implements FhirPath { * The HAPI object that represents the value of this literal. */ @Getter - protected Type literalValue; + protected ValueType value; + + @Nonnull + protected final Optional expression; protected LiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final ValueType value) { this.idColumn = idColumn; - this.literalValue = literalValue; + this.value = value; this.dataset = dataset; this.valueColumn = buildValueColumn(); + this.expression = Optional.empty(); + } + + protected LiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final ValueType value, @Nonnull final String expression) { + this.idColumn = idColumn; + this.value = value; + this.dataset = dataset; + this.valueColumn = buildValueColumn(); + this.expression = Optional.of(expression); } /** @@ -111,12 +124,19 @@ public static String expressionFor(@Nonnull final Dataset dataset, final Class literalPathClass = FHIR_TYPE_TO_FHIRPATH_TYPE .get(FHIRDefinedType.fromCode(literalValue.fhirType())); try { - final Constructor constructor = literalPathClass - .getDeclaredConstructor(Dataset.class, Column.class, Type.class); + @SuppressWarnings("unchecked") + final Constructor constructor = (Constructor) Arrays.stream( + literalPathClass.getDeclaredConstructors()) + .filter(c -> c.getParameterCount() == 3) + .filter(c -> c.getParameterTypes()[0] == Dataset.class) + .filter(c -> c.getParameterTypes()[1] == Column.class) + .filter(c -> Type.class.isAssignableFrom(c.getParameterTypes()[2])) + .findFirst() + .orElseThrow(() -> new AssertionException( + "No suitable constructor found for " + literalPathClass)); final LiteralPath literalPath = constructor.newInstance(dataset, idColumn, literalValue); return literalPath.getExpression(); - } catch (final NoSuchMethodException | InstantiationException | IllegalAccessException | - InvocationTargetException e) { + } catch (final InstantiationException | IllegalAccessException | InvocationTargetException e) { throw new RuntimeException("Problem building a LiteralPath class", e); } } @@ -152,21 +172,11 @@ public Column getExtractableColumn() { return getValueColumn(); } - /** - * Returns the Java object that represents the value of this literal. - * - * @return An Object - */ - @Nullable - public abstract Object getJavaValue(); - /** * @return A column representing the value for this literal. */ @Nonnull - public Column buildValueColumn() { - return lit(getJavaValue()); - } + public abstract Column buildValueColumn(); /** * @param fhirPathClass a subclass of LiteralPath diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/NullLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/NullLiteralPath.java index 8555f25166..9811580b11 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/NullLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/NullLiteralPath.java @@ -14,7 +14,6 @@ import au.csiro.pathling.fhirpath.FhirPath; import java.util.function.Function; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -25,11 +24,10 @@ * * @author John Grimes */ -public class NullLiteralPath extends LiteralPath implements Comparable { +public class NullLiteralPath extends LiteralPath implements Comparable { private static final String EXPRESSION = "{}"; - @SuppressWarnings("WeakerAccess") protected NullLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn) { // We put a dummy String value in here as a placeholder so that we can satisfy the nullability // constraints within LiteralValue. It is never accessed. @@ -57,10 +55,10 @@ public String getExpression() { return EXPRESSION; } - @Nullable + @Nonnull @Override - public Object getJavaValue() { - return null; + public Column buildValueColumn() { + return lit(null); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 3a7b92ac52..83848dc78a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -6,7 +6,6 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.struct; @@ -31,7 +30,6 @@ import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath Quantity literal. @@ -39,17 +37,21 @@ * @author John Grimes */ @Getter -public class QuantityLiteralPath extends LiteralPath implements Comparable, Numeric { +public class QuantityLiteralPath extends LiteralPath implements Comparable, Numeric { public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); - protected QuantityLiteralPath(@Nonnull final Dataset dataset, - @Nonnull final Column idColumn, @Nonnull final Type literalValue) { + protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final Quantity literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof Quantity); + } + + protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final Quantity literalValue, @Nonnull final String expression) { + super(dataset, idColumn, literalValue, expression); } /** @@ -72,7 +74,7 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, final String fullPath = matcher.group(0); final String value = matcher.group(1); final String rawUnit = matcher.group(2); - final String unit = StringLiteralPath.fromString(rawUnit, context).getLiteralValue() + final String unit = StringLiteralPath.fromString(rawUnit, context).getValue() .getValueAsString(); final String validationResult = ucumService.validate(unit); @@ -85,7 +87,7 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, final BigDecimal decimalValue = getQuantityValue(value, context); final String display = ucumService.getCommonDisplay(unit); - return buildLiteralPath(decimalValue, unit, display, context); + return buildLiteralPath(decimalValue, unit, display, context, fhirPath); } @Nonnull @@ -104,13 +106,13 @@ public static QuantityLiteralPath fromCalendarDurationString(@Nonnull final Stri quantity.setSystem(FHIRPATH_CALENDAR_DURATION_URI); quantity.setCode(keyword); - return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); + return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity, fhirPath); } private static BigDecimal getQuantityValue(final String value, final @Nonnull FhirPath context) { final BigDecimal decimalValue; try { - decimalValue = DecimalLiteralPath.fromString(value, context).getLiteralValue().getValue(); + decimalValue = DecimalLiteralPath.fromString(value, context).getValue().getValue(); } catch (final NumberFormatException e) { throw new IllegalArgumentException("Quantity literal has invalid value: " + value); } @@ -119,37 +121,28 @@ private static BigDecimal getQuantityValue(final String value, final @Nonnull Fh @Nonnull private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, - final String unit, final String display, final @Nonnull FhirPath context) { + final String unit, final String display, final @Nonnull FhirPath context, + final String fhirPath) { final Quantity quantity = new Quantity(); quantity.setValue(decimalValue); quantity.setSystem(Ucum.SYSTEM_URI); quantity.setCode(unit); quantity.setUnit(display); - return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity); + return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity, fhirPath); } @Nonnull @Override public String getExpression() { - return getLiteralValue().getValue().toPlainString() + " '" + getLiteralValue().getUnit() + "'"; - } - - @Override - public Quantity getLiteralValue() { - return (Quantity) literalValue; - } - - @Nonnull - @Override - public Quantity getJavaValue() { - return getLiteralValue(); + return expression.orElse( + getValue().getValue().toPlainString() + " '" + getValue().getUnit() + "'"); } @Nonnull @Override public Column buildValueColumn() { - final Quantity value = getJavaValue(); + final Quantity value = getValue(); final Optional comparator = Optional.ofNullable(value.getComparator()); return struct( lit(value.getId()).as("id"), diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java index 7cb5575d6f..9cd55706fe 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java @@ -6,8 +6,8 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.utilities.Preconditions.check; import static au.csiro.pathling.utilities.Strings.unSingleQuote; +import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -23,7 +23,6 @@ import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.PrimitiveType; import org.hl7.fhir.r4.model.StringType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath string literal. @@ -31,13 +30,12 @@ * @author John Grimes */ @Getter -public class StringLiteralPath extends LiteralPath implements Materializable, - Comparable { +public class StringLiteralPath extends LiteralPath implements + Materializable, Comparable { protected StringLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final PrimitiveType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof PrimitiveType); } /** @@ -74,13 +72,13 @@ public static String escapeFhirPathString(@Nonnull final String value) { @Nonnull @Override public String getExpression() { - return "'" + escapeFhirPathString(getLiteralValue().getValueAsString()) + "'"; + return "'" + escapeFhirPathString(getValue().getValueAsString()) + "'"; } @Nonnull @Override - public PrimitiveType getLiteralValue() { - return (PrimitiveType) literalValue; + public Column buildValueColumn() { + return lit(getValue().getValueAsString()); } /** @@ -102,12 +100,6 @@ public static String unescapeFhirPathString(@Nonnull String value) { return value.replaceAll("\\\\\\\\", "\\\\"); } - @Nonnull - @Override - public String getJavaValue() { - return getLiteralValue().getValueAsString(); - } - @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java index d70a6743e3..ca028b9d61 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java @@ -7,7 +7,7 @@ package au.csiro.pathling.fhirpath.literal; import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; -import static au.csiro.pathling.utilities.Preconditions.check; +import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -24,21 +24,23 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.TimeType; -import org.hl7.fhir.r4.model.Type; /** * Represents a FHIRPath time literal. * * @author John Grimes */ -public class TimeLiteralPath extends LiteralPath implements Materializable, Comparable, - Temporal { +public class TimeLiteralPath extends LiteralPath implements Materializable, + Comparable, Temporal { - @SuppressWarnings("WeakerAccess") protected TimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, - @Nonnull final Type literalValue) { + @Nonnull final TimeType literalValue) { super(dataset, idColumn, literalValue); - check(literalValue instanceof TimeType); + } + + protected TimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, + @Nonnull final TimeType literalValue, @Nonnull final String expression) { + super(dataset, idColumn, literalValue, expression); } /** @@ -54,24 +56,19 @@ public static TimeLiteralPath fromString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) { final String timeString = fhirPath.replaceFirst("^@T", ""); return new TimeLiteralPath(context.getDataset(), context.getIdColumn(), - new TimeType(timeString)); + new TimeType(timeString), fhirPath); } @Nonnull @Override public String getExpression() { - return "@T" + getLiteralValue().getValue(); - } - - @Override - public TimeType getLiteralValue() { - return (TimeType) literalValue; + return expression.orElse("@T" + getValue().getValue()); } @Nonnull @Override - public String getJavaValue() { - return getLiteralValue().getValue(); + public Column buildValueColumn() { + return lit(getValue().asStringValue()); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java index b13856bf52..7302983b3a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java @@ -48,7 +48,7 @@ public FhirPath invoke(@Nonnull final OperatorInput input) { checkUserInput(right instanceof QuantityLiteralPath, type + " operator does not support right operand: " + right.getExpression()); final QuantityLiteralPath calendarDuration = (QuantityLiteralPath) right; - checkUserInput(calendarDuration.getJavaValue().getSystem() + checkUserInput(calendarDuration.getValue().getSystem() .equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI), "Right operand of " + type + " operator must be a calendar duration"); checkUserInput(left.isSingular(), diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java index 1aba49e9f3..d890de21ea 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java @@ -6,7 +6,6 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.fhirpath.element.DateTimePath; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneOffset; @@ -27,7 +26,6 @@ Function parseEncodedValue() { Function encodeResult() { return (resultDate) -> new DateType( new Date(resultDate.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) * 1000)) - .setTimeZone(DateTimePath.getTimeZone()) .getValueAsString(); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java index 8f0a9bc479..77183ae688 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java @@ -7,9 +7,11 @@ package au.csiro.pathling.sql.dates; import au.csiro.pathling.fhirpath.element.DateTimePath; +import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import java.time.ZonedDateTime; import java.util.Date; import java.util.function.Function; +import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.DateTimeType; public abstract class DateTimeArithmeticFunction extends TemporalArithmeticFunction { @@ -23,9 +25,12 @@ Function parseEncodedValue() { @Override Function encodeResult() { - return (resultDateTime) -> new DateTimeType(new Date(resultDateTime.toEpochSecond() * 1000)) - .setTimeZone(DateTimePath.getTimeZone()) - .getValueAsString(); + return (resultDateTime) -> { + final BaseDateTimeType dateTime = new DateTimeType(Date.from(resultDateTime.toInstant())) + .setTimeZone(DateTimePath.getDefaultTimeZone()); + dateTime.setPrecision(TemporalPrecisionEnum.MILLI); + return dateTime.getValueAsString(); + }; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index ca12b8707b..235029f0eb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -6,6 +6,8 @@ package au.csiro.pathling.sql.dates; +import static au.csiro.pathling.utilities.Preconditions.checkNotNull; + import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.udf.SqlFunction2; @@ -44,6 +46,8 @@ public abstract class TemporalArithmeticFunction { private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() .put("second", "s") .put("seconds", "s") + .put("millisecond", "ms") + .put("milliseconds", "ms") .build(); /** diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPathTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPathTest.java index 779ab8ed66..f386769bad 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPathTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPathTest.java @@ -65,7 +65,7 @@ void roundTrip() { final CodingLiteralPath codingLiteralPath = CodingLiteralPath.fromString( expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("http://snomed.info/sct", literalValue.getSystem()); assertEquals("http://snomed.info/sct/32506021000036107/version/20201231", literalValue.getVersion()); @@ -80,7 +80,7 @@ void roundTripNoVersion() { final String expression = "http://snomed.info/sct|166056000"; final CodingLiteralPath codingLiteralPath = CodingLiteralPath .fromString(expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("http://snomed.info/sct", literalValue.getSystem()); assertNull(literalValue.getVersion()); assertEquals("166056000", literalValue.getCode()); @@ -98,7 +98,7 @@ void roundTripWithQuotedComponent() { + "|http://snomed.info/sct/32506021000036107/version/20201231"; final CodingLiteralPath codingLiteralPath = CodingLiteralPath .fromString(expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("http://snomed.info/sct", literalValue.getSystem()); assertEquals("http://snomed.info/sct/32506021000036107/version/20201231", literalValue.getVersion()); @@ -117,7 +117,7 @@ void roundTripWithQuotedComponentWithComma() { "http://snomed.info/sct|'46,2'|http://snomed.info/sct/32506021000036107/version/20201231"; final CodingLiteralPath codingLiteralPath = CodingLiteralPath .fromString(expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("http://snomed.info/sct", literalValue.getSystem()); assertEquals("http://snomed.info/sct/32506021000036107/version/20201231", literalValue.getVersion()); @@ -132,7 +132,7 @@ void roundTripWithQuotedComponentWithSingleQuote() { final String expression = "'Someone\\'s CodeSystem'|166056000"; final CodingLiteralPath codingLiteralPath = CodingLiteralPath .fromString(expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("Someone's CodeSystem", literalValue.getSystem()); assertEquals("166056000", literalValue.getCode()); @@ -145,7 +145,7 @@ void roundTripWithQuotedComponentWithSpace() { final String expression = "'Some CodeSystem'|166056000"; final CodingLiteralPath codingLiteralPath = CodingLiteralPath .fromString(expression, inputContext); - final Coding literalValue = codingLiteralPath.getLiteralValue(); + final Coding literalValue = codingLiteralPath.getValue(); assertEquals("Some CodeSystem", literalValue.getSystem()); assertEquals("166056000", literalValue.getCode()); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index 3f8b906ea4..d390558f76 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -88,7 +88,7 @@ Stream parameters() throws ParseException { final Dataset dateTimeDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) .withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T13:28:17-05:00") + .withRow("patient-1", "2015-02-07T13:28:17.000-05:00") .withRow("patient-2", "2017-01-01T00:00:00+00:00") .withRow("patient-3", "2025-06-21T00:15:00+10:00") .build(); @@ -159,9 +159,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17+00:00") - .withRow("patient-2", "2027-01-01T00:00:00+00:00") - .withRow("patient-3", "2035-06-20T14:15:00+00:00") + .withRow("patient-1", "2025-02-07T18:28:17.000+00:00") + .withRow("patient-2", "2027-01-01T00:00:00.000+00:00") + .withRow("patient-3", "2035-06-20T14:15:00.000+00:00") .build()) ); @@ -169,9 +169,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17+00:00") - .withRow("patient-2", "2017-10-01T00:00:00+00:00") - .withRow("patient-3", "2026-03-20T14:15:00+00:00") + .withRow("patient-1", "2015-11-07T18:28:17.000+00:00") + .withRow("patient-2", "2017-10-01T00:00:00.000+00:00") + .withRow("patient-3", "2026-03-20T14:15:00.000+00:00") .build()) ); @@ -179,9 +179,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17+00:00") - .withRow("patient-2", "2017-01-15T00:00:00+00:00") - .withRow("patient-3", "2025-07-04T14:15:00+00:00") + .withRow("patient-1", "2015-02-21T18:28:17.000+00:00") + .withRow("patient-2", "2017-01-15T00:00:00.000+00:00") + .withRow("patient-3", "2025-07-04T14:15:00.000+00:00") .build()) ); @@ -189,9 +189,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17+00:00") - .withRow("patient-2", "2017-01-31T00:00:00+00:00") - .withRow("patient-3", "2025-07-20T14:15:00+00:00") + .withRow("patient-1", "2015-03-09T18:28:17.000+00:00") + .withRow("patient-2", "2017-01-31T00:00:00.000+00:00") + .withRow("patient-3", "2025-07-20T14:15:00.000+00:00") .build()) ); @@ -199,9 +199,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17+00:00") - .withRow("patient-2", "2017-01-01T12:00:00+00:00") - .withRow("patient-3", "2025-06-21T02:15:00+00:00") + .withRow("patient-1", "2015-02-08T06:28:17.000+00:00") + .withRow("patient-2", "2017-01-01T12:00:00.000+00:00") + .withRow("patient-3", "2025-06-21T02:15:00.000+00:00") .build()) ); @@ -209,9 +209,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17+00:00") - .withRow("patient-2", "2017-01-01T00:30:00+00:00") - .withRow("patient-3", "2025-06-20T14:45:00+00:00") + .withRow("patient-1", "2015-02-07T18:58:17.000+00:00") + .withRow("patient-2", "2017-01-01T00:30:00.000+00:00") + .withRow("patient-3", "2025-06-20T14:45:00.000+00:00") .build()) ); @@ -219,12 +219,21 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27+00:00") - .withRow("patient-2", "2017-01-01T00:00:10+00:00") - .withRow("patient-3", "2025-06-20T14:15:10+00:00") + .withRow("patient-1", "2015-02-07T18:28:27.000+00:00") + .withRow("patient-2", "2017-01-01T00:00:10.000+00:00") + .withRow("patient-3", "2025-06-20T14:15:10.000+00:00") .build()) ); + parameters.add(new TestParameters("DateTime + 300 milliseconds", dateTimePath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300+00:00") + .withRow("patient-2", "2017-01-01T00:00:00.300+00:00") + .withRow("patient-3", "2025-06-20T14:15:00.300+00:00") + .build()) + ); return parameters; } @@ -235,9 +244,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17+00:00") - .withRow("patient-2", "2007-01-01T00:00:00+00:00") - .withRow("patient-3", "2015-06-20T14:15:00+00:00") + .withRow("patient-1", "2005-02-07T18:28:17.000+00:00") + .withRow("patient-2", "2007-01-01T00:00:00.000+00:00") + .withRow("patient-3", "2015-06-20T14:15:00.000+00:00") .build()) ); @@ -245,9 +254,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17+00:00") - .withRow("patient-2", "2016-04-01T00:00:00+00:00") - .withRow("patient-3", "2024-09-20T14:15:00+00:00") + .withRow("patient-1", "2014-05-07T18:28:17.000+00:00") + .withRow("patient-2", "2016-04-01T00:00:00.000+00:00") + .withRow("patient-3", "2024-09-20T14:15:00.000+00:00") .build()) ); @@ -255,9 +264,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17+00:00") - .withRow("patient-2", "2016-12-18T00:00:00+00:00") - .withRow("patient-3", "2025-06-06T14:15:00+00:00") + .withRow("patient-1", "2015-01-24T18:28:17.000+00:00") + .withRow("patient-2", "2016-12-18T00:00:00.000+00:00") + .withRow("patient-3", "2025-06-06T14:15:00.000+00:00") .build()) ); @@ -265,9 +274,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17+00:00") - .withRow("patient-2", "2016-12-02T00:00:00+00:00") - .withRow("patient-3", "2025-05-21T14:15:00+00:00") + .withRow("patient-1", "2015-01-08T18:28:17.000+00:00") + .withRow("patient-2", "2016-12-02T00:00:00.000+00:00") + .withRow("patient-3", "2025-05-21T14:15:00.000+00:00") .build()) ); @@ -275,9 +284,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17+00:00") - .withRow("patient-2", "2016-12-31T12:00:00+00:00") - .withRow("patient-3", "2025-06-20T02:15:00+00:00") + .withRow("patient-1", "2015-02-07T06:28:17.000+00:00") + .withRow("patient-2", "2016-12-31T12:00:00.000+00:00") + .withRow("patient-3", "2025-06-20T02:15:00.000+00:00") .build()) ); @@ -285,9 +294,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17+00:00") - .withRow("patient-2", "2016-12-31T23:30:00+00:00") - .withRow("patient-3", "2025-06-20T13:45:00+00:00") + .withRow("patient-1", "2015-02-07T17:58:17.000+00:00") + .withRow("patient-2", "2016-12-31T23:30:00.000+00:00") + .withRow("patient-3", "2025-06-20T13:45:00.000+00:00") .build()) ); @@ -295,12 +304,21 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07+00:00") - .withRow("patient-2", "2016-12-31T23:59:50+00:00") - .withRow("patient-3", "2025-06-20T14:14:50+00:00") + .withRow("patient-1", "2015-02-07T18:28:07.000+00:00") + .withRow("patient-2", "2016-12-31T23:59:50.000+00:00") + .withRow("patient-3", "2025-06-20T14:14:50.000+00:00") .build()) ); + parameters.add(new TestParameters("DateTime - 300 milliseconds", dateTimePath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700+00:00") + .withRow("patient-2", "2016-12-31T23:59:59.700+00:00") + .withRow("patient-3", "2025-06-20T14:14:59.700+00:00") + .build()) + ); return parameters; } @@ -403,9 +421,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17+00:00") - .withRow("patient-2", "2025-02-07T18:28:17+00:00") - .withRow("patient-3", "2025-02-07T18:28:17+00:00") + .withRow("patient-1", "2025-02-07T18:28:17.000+00:00") + .withRow("patient-2", "2025-02-07T18:28:17.000+00:00") + .withRow("patient-3", "2025-02-07T18:28:17.000+00:00") .build()) ); @@ -413,9 +431,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17+00:00") - .withRow("patient-2", "2015-11-07T18:28:17+00:00") - .withRow("patient-3", "2015-11-07T18:28:17+00:00") + .withRow("patient-1", "2015-11-07T18:28:17.000+00:00") + .withRow("patient-2", "2015-11-07T18:28:17.000+00:00") + .withRow("patient-3", "2015-11-07T18:28:17.000+00:00") .build()) ); @@ -423,9 +441,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17+00:00") - .withRow("patient-2", "2015-02-21T18:28:17+00:00") - .withRow("patient-3", "2015-02-21T18:28:17+00:00") + .withRow("patient-1", "2015-02-21T18:28:17.000+00:00") + .withRow("patient-2", "2015-02-21T18:28:17.000+00:00") + .withRow("patient-3", "2015-02-21T18:28:17.000+00:00") .build()) ); @@ -433,9 +451,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17+00:00") - .withRow("patient-2", "2015-03-09T18:28:17+00:00") - .withRow("patient-3", "2015-03-09T18:28:17+00:00") + .withRow("patient-1", "2015-03-09T18:28:17.000+00:00") + .withRow("patient-2", "2015-03-09T18:28:17.000+00:00") + .withRow("patient-3", "2015-03-09T18:28:17.000+00:00") .build()) ); @@ -443,9 +461,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17+00:00") - .withRow("patient-2", "2015-02-08T06:28:17+00:00") - .withRow("patient-3", "2015-02-08T06:28:17+00:00") + .withRow("patient-1", "2015-02-08T06:28:17.000+00:00") + .withRow("patient-2", "2015-02-08T06:28:17.000+00:00") + .withRow("patient-3", "2015-02-08T06:28:17.000+00:00") .build()) ); @@ -455,9 +473,9 @@ Collection dateTimeLiteralAddition( context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17+00:00") - .withRow("patient-2", "2015-02-07T18:58:17+00:00") - .withRow("patient-3", "2015-02-07T18:58:17+00:00") + .withRow("patient-1", "2015-02-07T18:58:17.000+00:00") + .withRow("patient-2", "2015-02-07T18:58:17.000+00:00") + .withRow("patient-3", "2015-02-07T18:58:17.000+00:00") .build()) ); @@ -467,9 +485,21 @@ Collection dateTimeLiteralAddition( context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27+00:00") - .withRow("patient-2", "2015-02-07T18:28:27+00:00") - .withRow("patient-3", "2015-02-07T18:28:27+00:00") + .withRow("patient-1", "2015-02-07T18:28:27.000+00:00") + .withRow("patient-2", "2015-02-07T18:28:27.000+00:00") + .withRow("patient-3", "2015-02-07T18:28:27.000+00:00") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17+00:00 + 300 milliseconds", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), + context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:17.300+00:00") + .withRow("patient-2", "2015-02-07T18:28:17.300+00:00") + .withRow("patient-3", "2015-02-07T18:28:17.300+00:00") .build()) ); @@ -483,9 +513,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17+00:00") - .withRow("patient-2", "2005-02-07T18:28:17+00:00") - .withRow("patient-3", "2005-02-07T18:28:17+00:00") + .withRow("patient-1", "2005-02-07T18:28:17.000+00:00") + .withRow("patient-2", "2005-02-07T18:28:17.000+00:00") + .withRow("patient-3", "2005-02-07T18:28:17.000+00:00") .build()) ); @@ -493,9 +523,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17+00:00") - .withRow("patient-2", "2014-05-07T18:28:17+00:00") - .withRow("patient-3", "2014-05-07T18:28:17+00:00") + .withRow("patient-1", "2014-05-07T18:28:17.000+00:00") + .withRow("patient-2", "2014-05-07T18:28:17.000+00:00") + .withRow("patient-3", "2014-05-07T18:28:17.000+00:00") .build()) ); @@ -503,9 +533,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17+00:00") - .withRow("patient-2", "2015-01-24T18:28:17+00:00") - .withRow("patient-3", "2015-01-24T18:28:17+00:00") + .withRow("patient-1", "2015-01-24T18:28:17.000+00:00") + .withRow("patient-2", "2015-01-24T18:28:17.000+00:00") + .withRow("patient-3", "2015-01-24T18:28:17.000+00:00") .build()) ); @@ -513,9 +543,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17+00:00") - .withRow("patient-2", "2015-01-08T18:28:17+00:00") - .withRow("patient-3", "2015-01-08T18:28:17+00:00") + .withRow("patient-1", "2015-01-08T18:28:17.000+00:00") + .withRow("patient-2", "2015-01-08T18:28:17.000+00:00") + .withRow("patient-3", "2015-01-08T18:28:17.000+00:00") .build()) ); @@ -523,9 +553,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17+00:00") - .withRow("patient-2", "2015-02-07T06:28:17+00:00") - .withRow("patient-3", "2015-02-07T06:28:17+00:00") + .withRow("patient-1", "2015-02-07T06:28:17.000+00:00") + .withRow("patient-2", "2015-02-07T06:28:17.000+00:00") + .withRow("patient-3", "2015-02-07T06:28:17.000+00:00") .build()) ); @@ -535,9 +565,9 @@ Collection dateTimeLiteralSubtraction( context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17+00:00") - .withRow("patient-2", "2015-02-07T17:58:17+00:00") - .withRow("patient-3", "2015-02-07T17:58:17+00:00") + .withRow("patient-1", "2015-02-07T17:58:17.000+00:00") + .withRow("patient-2", "2015-02-07T17:58:17.000+00:00") + .withRow("patient-3", "2015-02-07T17:58:17.000+00:00") .build()) ); @@ -547,9 +577,21 @@ Collection dateTimeLiteralSubtraction( context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07+00:00") - .withRow("patient-2", "2015-02-07T18:28:07+00:00") - .withRow("patient-3", "2015-02-07T18:28:07+00:00") + .withRow("patient-1", "2015-02-07T18:28:07.000+00:00") + .withRow("patient-2", "2015-02-07T18:28:07.000+00:00") + .withRow("patient-3", "2015-02-07T18:28:07.000+00:00") + .build()) + ); + + parameters.add( + new TestParameters("@2015-02-07T18:28:17+00:00 - 300 milliseconds", dateTimeLiteralPath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimeLiteralPath), + context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "2015-02-07T18:28:16.700+00:00") + .withRow("patient-2", "2015-02-07T18:28:16.700+00:00") + .withRow("patient-3", "2015-02-07T18:28:16.700+00:00") .build()) ); @@ -677,6 +719,15 @@ Collection timeAddition(final FhirPath timePath, .withRow("patient-2", "08:00:15") .withRow("patient-3", "00:00:15") .build())); + + parameters.add(new TestParameters("Time + 300 milliseconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "13:28:17.300") + .withRow("patient-2", "08:00:00.300") + .withRow("patient-3", "00:00:00.300") + .build())); return parameters; } @@ -709,6 +760,15 @@ Collection timeSubtraction(final FhirPath timePath, .withRow("patient-2", "07:59:45") .withRow("patient-3", "23:59:45") .build())); + + parameters.add(new TestParameters("Time - 300 milliseconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "13:28:16.700") + .withRow("patient-2", "07:59:59.700") + .withRow("patient-3", "23:59:59.700") + .build())); return parameters; } @@ -741,6 +801,15 @@ Collection timeLiteralAddition(final FhirPath timePath, .withRow("patient-2", "08:00:15") .withRow("patient-3", "08:00:15") .build())); + + parameters.add(new TestParameters("@T08:00 + 70 milliseconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("70 milliseconds", timePath), context, + Operator.getInstance("+"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "08:00:00.070") + .withRow("patient-2", "08:00:00.070") + .withRow("patient-3", "08:00:00.070") + .build())); return parameters; } @@ -773,6 +842,15 @@ Collection timeLiteralSubtraction(final FhirPath timePath, .withRow("patient-2", "07:59:45") .withRow("patient-3", "07:59:45") .build())); + + parameters.add(new TestParameters("@T08:00 - 70 milliseconds", timePath, + QuantityLiteralPath.fromCalendarDurationString("70 milliseconds", timePath), context, + Operator.getInstance("-"), + new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) + .withRow("patient-1", "07:59:59.930") + .withRow("patient-2", "07:59:59.930") + .withRow("patient-3", "07:59:59.930") + .build())); return parameters; } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java index 9e721b592d..41e0f41e4b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java @@ -47,7 +47,6 @@ import au.csiro.pathling.test.fixtures.ConceptTranslatorBuilder; import au.csiro.pathling.test.fixtures.RelationBuilder; import au.csiro.pathling.test.helpers.TerminologyHelpers; -import java.sql.Date; import java.util.Collections; import org.apache.spark.sql.types.DataTypes; import org.hl7.fhir.r4.model.Coding; @@ -126,32 +125,80 @@ void testCodingOperations() { @Test void testDateTimeLiterals() { - // Full DateTime. - assertThatResultOf("@2015-02-04T14:34:28Z") + // Milliseconds precision. + assertThatResultOf("@2015-02-04T14:34:28.350Z") .isLiteralPath(DateTimeLiteralPath.class) - .hasExpression("@2015-02-04T14:34:28Z") - .hasJavaValue(new Date(1423060468000L)); + .hasExpression("@2015-02-04T14:34:28.350Z") + .has("2015-02-04T14:34:28.350Z", + dateTime -> dateTime.getValue().dateTimeValue().getValueAsString()); - // Date with no time component. + // Milliseconds precision, no timezone. + assertThatResultOf("@2015-02-04T14:34:28.350") + .isLiteralPath(DateTimeLiteralPath.class) + .hasExpression("@2015-02-04T14:34:28.350") + .has("2015-02-04T14:34:28.350", + dateTime -> dateTime.getValue().dateTimeValue().getValueAsString()) + .has(null, dateTime -> dateTime.getValue().dateTimeValue().getTimeZone()); + + // Seconds precision. + assertThatResultOf("@2015-02-04T14:34:28-05:00") + .isLiteralPath(DateTimeLiteralPath.class) + .hasExpression("@2015-02-04T14:34:28-05:00") + .has("2015-02-04T14:34:28-05:00", + dateTime -> dateTime.getValue().dateTimeValue().getValueAsString()); + + // Seconds precision, no timezone. + assertThatResultOf("@2015-02-04T14:34:28") + .isLiteralPath(DateTimeLiteralPath.class) + .hasExpression("@2015-02-04T14:34:28") + .has("2015-02-04T14:34:28", + dateTime -> dateTime.getValue().dateTimeValue().getValueAsString()) + .has(null, dateTime -> dateTime.getValue().dateTimeValue().getTimeZone()); + } + + @Test + void testDateLiterals() { + // Year, month and day. assertThatResultOf("@2015-02-04") .isLiteralPath(DateLiteralPath.class) .hasExpression("@2015-02-04") - .hasJavaValue(new Date(1423008000000L)); + .has("2015-02-04", date -> date.getValue().castToDate(date.getValue()).getValueAsString()) + .has(null, date -> date.getValue().castToDate(date.getValue()).getTimeZone()); + + // Year and month. + assertThatResultOf("@2015-02") + .isLiteralPath(DateLiteralPath.class) + .hasExpression("@2015-02") + .has("2015-02", date -> date.getValue().castToDate(date.getValue()).getValueAsString()) + .has(null, date -> date.getValue().castToDate(date.getValue()).getTimeZone()); + + // Year only. + assertThatResultOf("@2015") + .isLiteralPath(DateLiteralPath.class) + .hasExpression("@2015") + .has("2015", date -> date.getValue().castToDate(date.getValue()).getValueAsString()) + .has(null, date -> date.getValue().castToDate(date.getValue()).getTimeZone()); } @Test void testTimeLiterals() { - // Full Time. + // Hours, minutes and seconds. assertThatResultOf("@T14:34:28") .isLiteralPath(TimeLiteralPath.class) .hasExpression("@T14:34:28") - .hasJavaValue("14:34:28"); + .has("14:34:28", time -> time.getValue().castToTime(time.getValue()).getValueAsString()); + + // Hours and minutes. + assertThatResultOf("@T14:34") + .isLiteralPath(TimeLiteralPath.class) + .hasExpression("@T14:34") + .has("14:34", time -> time.getValue().castToTime(time.getValue()).getValueAsString()); // Hour only. assertThatResultOf("@T14") .isLiteralPath(TimeLiteralPath.class) .hasExpression("@T14") - .hasJavaValue("14"); + .has("14", time -> time.getValue().castToTime(time.getValue()).getValueAsString()); } @Test diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 126100d5ef..cc3731a310 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -20,6 +20,7 @@ import au.csiro.pathling.sql.dates.SubtractDurationFromTime; import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; +import au.csiro.pathling.terminology.CodingToLiteral; import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.terminology.ucum.ComparableQuantity; import au.csiro.pathling.terminology.ucum.Ucum; @@ -52,7 +53,8 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, @Nonnull final UcumService ucumService) { - final List sqlFunction1 = List.of(new ComparableQuantity(ucumService)); + final List sqlFunction1 = List.of(new CodingToLiteral(), + new ComparableQuantity(ucumService)); final List sqlFunction2 = List.of(new AddDurationToDateTime(), new SubtractDurationFromDateTime(), new AddDurationToDate(), new SubtractDurationFromDate(), new AddDurationToTime(), new SubtractDurationFromTime()); diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/assertions/LiteralPathAssertion.java b/fhir-server/src/test/java/au/csiro/pathling/test/assertions/LiteralPathAssertion.java index 82b30511cb..707391bfcb 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/assertions/LiteralPathAssertion.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/assertions/LiteralPathAssertion.java @@ -12,7 +12,9 @@ import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; import au.csiro.pathling.fhirpath.literal.LiteralPath; +import java.util.function.Function; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.hl7.fhir.r4.model.Coding; /** @@ -30,8 +32,9 @@ public class LiteralPathAssertion extends BaseFhirPathAssertion function) { + assertEquals(expected, function.apply(fhirPath)); return this; } @@ -39,7 +42,7 @@ public LiteralPathAssertion hasJavaValue(@Nonnull final Object value) { public LiteralPathAssertion hasCodingValue(@Nonnull final Coding coding) { assertTrue(fhirPath instanceof CodingLiteralPath); final SimpleCoding actualCoding = new SimpleCoding( - ((CodingLiteralPath) fhirPath).getJavaValue()); + ((CodingLiteralPath) fhirPath).getValue()); final SimpleCoding expectedCoding = new SimpleCoding(coding); assertEquals(expectedCoding, actualCoding); return this; diff --git a/fhir-server/src/test/resources/responses/AggregateQueryTest/queryWithDateTimeGrouping.Parameters.json b/fhir-server/src/test/resources/responses/AggregateQueryTest/queryWithDateTimeGrouping.Parameters.json index 961328ea38..0204efaf02 100644 --- a/fhir-server/src/test/resources/responses/AggregateQueryTest/queryWithDateTimeGrouping.Parameters.json +++ b/fhir-server/src/test/resources/responses/AggregateQueryTest/queryWithDateTimeGrouping.Parameters.json @@ -14,7 +14,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2010-06-19T18:35:39Z" + "valueString": "(authoredOn) = @2010-06-19T18:35:39+00:00" } ] }, @@ -31,7 +31,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1964-10-16T09:11:55Z" + "valueString": "(authoredOn) = @1964-10-16T09:11:55+00:00" } ] }, @@ -48,7 +48,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1959-02-26T08:27:40Z" + "valueString": "(authoredOn) = @1959-02-26T08:27:40+00:00" } ] }, @@ -65,7 +65,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2000-06-28T09:11:55Z" + "valueString": "(authoredOn) = @2000-06-28T09:11:55+00:00" } ] }, @@ -82,7 +82,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2016-12-12T19:04:32Z" + "valueString": "(authoredOn) = @2016-12-12T19:04:32+00:00" } ] }, @@ -99,7 +99,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2002-06-18T09:11:55Z" + "valueString": "(authoredOn) = @2002-06-18T09:11:55+00:00" } ] }, @@ -116,7 +116,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1959-05-29T08:27:40Z" + "valueString": "(authoredOn) = @1959-05-29T08:27:40+00:00" } ] }, @@ -133,7 +133,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2017-02-18T19:04:32Z" + "valueString": "(authoredOn) = @2017-02-18T19:04:32+00:00" } ] }, @@ -150,7 +150,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2017-11-18T19:04:32Z" + "valueString": "(authoredOn) = @2017-11-18T19:04:32+00:00" } ] }, @@ -167,7 +167,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1960-02-21T08:27:40Z" + "valueString": "(authoredOn) = @1960-02-21T08:27:40+00:00" } ] }, @@ -184,7 +184,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1961-04-07T08:27:40Z" + "valueString": "(authoredOn) = @1961-04-07T08:27:40+00:00" } ] }, @@ -201,7 +201,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2005-07-10T09:11:55Z" + "valueString": "(authoredOn) = @2005-07-10T09:11:55+00:00" } ] }, @@ -218,7 +218,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1961-09-04T08:27:40Z" + "valueString": "(authoredOn) = @1961-09-04T08:27:40+00:00" } ] }, @@ -235,7 +235,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2015-06-02T09:24:41Z" + "valueString": "(authoredOn) = @2015-06-02T09:24:41+00:00" } ] }, @@ -252,7 +252,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2006-05-28T09:11:55Z" + "valueString": "(authoredOn) = @2006-05-28T09:11:55+00:00" } ] }, @@ -269,7 +269,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1965-07-15T08:27:40Z" + "valueString": "(authoredOn) = @1965-07-15T08:27:40+00:00" } ] }, @@ -286,7 +286,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2007-05-23T09:11:55Z" + "valueString": "(authoredOn) = @2007-05-23T09:11:55+00:00" } ] }, @@ -303,7 +303,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1966-06-09T08:27:40Z" + "valueString": "(authoredOn) = @1966-06-09T08:27:40+00:00" } ] }, @@ -320,7 +320,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1960-06-26T08:27:40Z" + "valueString": "(authoredOn) = @1960-06-26T08:27:40+00:00" } ] }, @@ -337,7 +337,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2010-10-03T08:27:40Z" + "valueString": "(authoredOn) = @2010-10-03T08:27:40+00:00" } ] }, @@ -354,7 +354,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1961-10-03T09:11:55Z" + "valueString": "(authoredOn) = @1961-10-03T09:11:55+00:00" } ] }, @@ -371,7 +371,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2008-02-11T09:11:55Z" + "valueString": "(authoredOn) = @2008-02-11T09:11:55+00:00" } ] }, @@ -388,7 +388,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2009-02-05T09:11:55Z" + "valueString": "(authoredOn) = @2009-02-05T09:11:55+00:00" } ] }, @@ -405,7 +405,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2011-11-02T09:11:55Z" + "valueString": "(authoredOn) = @2011-11-02T09:11:55+00:00" } ] }, @@ -422,7 +422,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2013-10-16T09:11:55Z" + "valueString": "(authoredOn) = @2013-10-16T09:11:55+00:00" } ] }, @@ -439,7 +439,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2013-10-23T09:11:55Z" + "valueString": "(authoredOn) = @2013-10-23T09:11:55+00:00" } ] }, @@ -456,7 +456,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2016-03-16T09:11:55Z" + "valueString": "(authoredOn) = @2016-03-16T09:11:55+00:00" } ] }, @@ -473,7 +473,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2016-09-07T09:11:55Z" + "valueString": "(authoredOn) = @2016-09-07T09:11:55+00:00" } ] }, @@ -490,7 +490,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1980-01-13T09:11:55Z" + "valueString": "(authoredOn) = @1980-01-13T09:11:55+00:00" } ] }, @@ -507,7 +507,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2001-08-25T09:11:55Z" + "valueString": "(authoredOn) = @2001-08-25T09:11:55+00:00" } ] }, @@ -524,7 +524,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2004-05-26T09:11:55Z" + "valueString": "(authoredOn) = @2004-05-26T09:11:55+00:00" } ] }, @@ -541,7 +541,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2005-05-21T09:11:55Z" + "valueString": "(authoredOn) = @2005-05-21T09:11:55+00:00" } ] }, @@ -558,7 +558,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2006-05-16T09:11:55Z" + "valueString": "(authoredOn) = @2006-05-16T09:11:55+00:00" } ] }, @@ -575,7 +575,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2007-05-11T09:11:55Z" + "valueString": "(authoredOn) = @2007-05-11T09:11:55+00:00" } ] }, @@ -592,7 +592,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2008-05-05T09:11:55Z" + "valueString": "(authoredOn) = @2008-05-05T09:11:55+00:00" } ] }, @@ -609,7 +609,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2008-06-18T09:11:55Z" + "valueString": "(authoredOn) = @2008-06-18T09:11:55+00:00" } ] }, @@ -626,7 +626,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2009-05-17T09:11:55Z" + "valueString": "(authoredOn) = @2009-05-17T09:11:55+00:00" } ] }, @@ -643,7 +643,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2009-09-29T09:11:55Z" + "valueString": "(authoredOn) = @2009-09-29T09:11:55+00:00" } ] }, @@ -660,7 +660,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2009-10-18T09:11:55Z" + "valueString": "(authoredOn) = @2009-10-18T09:11:55+00:00" } ] }, @@ -677,7 +677,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2010-10-18T09:11:55Z" + "valueString": "(authoredOn) = @2010-10-18T09:11:55+00:00" } ] }, @@ -694,7 +694,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @1977-11-20T09:11:55Z" + "valueString": "(authoredOn) = @1977-11-20T09:11:55+00:00" } ] }, @@ -711,7 +711,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2015-10-18T09:11:55Z" + "valueString": "(authoredOn) = @2015-10-18T09:11:55+00:00" } ] }, @@ -728,7 +728,7 @@ }, { "name": "drillDown", - "valueString": "(authoredOn) = @2019-04-14T09:11:55Z" + "valueString": "(authoredOn) = @2019-04-14T09:11:55+00:00" } ] } diff --git a/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json b/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json index 159c7ca123..d34c1cffd6 100644 --- a/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json +++ b/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json @@ -14,9 +14,9 @@ }, { "name": "drillDown", - "valueString": "(@2015-02-08T18:28:17Z) = @2015-02-08T18:28:17Z" + "valueString": "(@2015-02-08T13:28:17-05:00) = @2015-02-08T18:28:17+00:00" } ] } ] -} \ No newline at end of file +} diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index d4a015aebd..89cb6b67a4 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -58,7 +58,7 @@ Examples: ``` 'test string' 'urn:oid:3.4.5.6.7.8' -'M\u00fcller' +'M\u00fcller' // Includes a Unicode character, evaluates to Müller ``` ## Integer @@ -99,9 +99,9 @@ format, though month and day parts are optional. Examples: ``` -@2014-01-25 -@2014-01 -@2014 +@2014-01-25 // Year, month and day +@2014-01 // Year and month only +@2014 // Year only ``` ## Time @@ -118,21 +118,28 @@ The Time literal uses a subset of Examples: ``` -@T07:30:14 -@T14:30 -@T14 +@T07:30:14 // Seconds precision +@T14:30:14 // Minutes precision +@T14 // Hours precision ``` ## DateTime The DateTime literal combines the [Date](#date) and [Time](#time) literals and is a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). It uses the -`YYYY-MM-DDThh:mm:ss±hh:mm` format. Milliseconds are not supported. +`YYYY-MM-DDThh:mm:ss.ffff±hh:mm` format. `Z` is allowed as a synonym for the +zero (`+00:00`) UTC offset. + +Time zone is optional. Seconds and milliseconds precision are supported. Hours +precision and partial DateTime values (ending with `T`) are not supported. Example: ``` -@2015-02-08T13:28:17-05:00 +@2014-01-25T14:30:14 // Seconds precision +@2014-01-25T14:30:14+10:00 // Seconds precision with UTC+10 timezone offset +@2014-01-25T14:30:14.559 // Milliseconds precision +@2014-01-25T14:30:14.559Z // Milliseconds precision with UTC timezone offset ``` ## Quantity @@ -157,14 +164,15 @@ The calendar duration keywords that are supported are: - `hour` / `hours` - `minute` / `minutes` - `second` / `seconds` +- `millisecond` / `milliseconds` Example: ``` -4.5 'mg' -100 '[degF]' -6 months -30 days +4.5 'mg' // UCUM Quantity of 4.5 mg +100 '[degF]' // UCUM Quantity of 100 degrees Fahrenheit +6 months // Calendar duration of 6 months +30 days // Calendar duration of 30 days ``` See: [Quantity](https://hl7.org/fhirpath/#quantity) diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index 02f0c9e8a2..e1d5d8d527 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -42,8 +42,8 @@ combinations. | String | false | true | false | false | false | false | false | false | | Integer | false | false | true | true | false | false | false | false | | Decimal | false | false | true | true | false | false | false | false | -| Date | false | false | false | false | true | true | false | true* | -| DateTime | false | false | false | false | true | true | false | true* | +| Date | false | false | false | false | true | true | false | false | +| DateTime | false | false | false | false | true | true | false | false | | Time | false | false | false | false | false | false | true | false | | Quantity | false | false | false | false | false | false | false | true* | @@ -55,8 +55,8 @@ individual characters. All comparison operators return a [Boolean](./data-types.html#boolean) value. -* Not all Quantity, Date and DateTime values are comparable, it -depends upon the comparability of the units within the Quantity values. See the +* Not all Quantity values are comparable, it depends upon the +comparability of the units. See the [FHIRPath specification](https://hl7.org/fhirpath/#comparison) for details on how Quantity values are compared. Quantities with a `comparator` are treated as not comparable by this implementation. @@ -117,7 +117,8 @@ The following operators are supported for date arithmetic: - `-` - Subtract a duration from a [Date](./data-types.html#date), [DateTime](./data-types.html#datetime) or [Time](./data-types.html#time) -The duration operand is a +Date arithmetic always has a `DateTime`, `Date` or `Time` on the left-hand side, +and a duration on the right-hand side. The duration operand is a [calendar duration literal](./data-types.html#quantity). The use of UCUM units is not supported with these operators. From 8017b831e38f5ae7e09bc4a0b9b6ace05bf255a4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Apr 2022 16:52:50 +1000 Subject: [PATCH 15/83] Remove duplicate SqlFunction methods --- .../au/csiro/pathling/sql/udf/SqlFunction.java | 17 +++++++++++++++++ .../au/csiro/pathling/sql/udf/SqlFunction1.java | 7 +------ .../au/csiro/pathling/sql/udf/SqlFunction2.java | 7 +------ 3 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java new file mode 100644 index 0000000000..5f4735f94a --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java @@ -0,0 +1,17 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.udf; + +import org.apache.spark.sql.types.DataType; + +public interface SqlFunction { + + String getName(); + + DataType getReturnType(); + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java index 3d14307d90..8ecf9f8755 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java @@ -7,12 +7,7 @@ package au.csiro.pathling.sql.udf; import org.apache.spark.sql.api.java.UDF1; -import org.apache.spark.sql.types.DataType; -public interface SqlFunction1 extends UDF1 { - - String getName(); - - DataType getReturnType(); +public interface SqlFunction1 extends SqlFunction, UDF1 { } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java index ce26dba6f2..cdff936922 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java @@ -7,12 +7,7 @@ package au.csiro.pathling.sql.udf; import org.apache.spark.sql.api.java.UDF2; -import org.apache.spark.sql.types.DataType; -public interface SqlFunction2 extends UDF2 { - - String getName(); - - DataType getReturnType(); +public interface SqlFunction2 extends SqlFunction, UDF2 { } From 11c9aadeaea20d8ec7083da33ab691fb17d20d68 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Apr 2022 19:26:41 +1000 Subject: [PATCH 16/83] Add ComparisonOperatorDateTimeTest --- .../operator/ComparisonOperatorDateTest.java | 116 ---------- .../ComparisonOperatorDateTimeTest.java | 201 ++++++++++++++++++ 2 files changed, 201 insertions(+), 116 deletions(-) delete mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java deleted file mode 100644 index 57e83513e2..0000000000 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.fhirpath.operator; - -import static au.csiro.pathling.test.assertions.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import au.csiro.pathling.fhirpath.FhirPath; -import au.csiro.pathling.fhirpath.element.BooleanPath; -import au.csiro.pathling.fhirpath.element.ElementDefinition; -import au.csiro.pathling.fhirpath.element.ElementPath; -import au.csiro.pathling.fhirpath.parser.ParserContext; -import au.csiro.pathling.test.builders.DatasetBuilder; -import au.csiro.pathling.test.builders.ElementPathBuilder; -import au.csiro.pathling.test.builders.ParserContextBuilder; -import au.csiro.pathling.test.helpers.FhirHelpers; -import ca.uhn.fhir.context.FhirContext; -import java.util.Collections; -import java.util.Optional; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.types.DataTypes; -import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; - -/** - * @author John Grimes - */ -@SpringBootTest -@Tag("UnitTest") -class ComparisonOperatorDateTest { - - @Autowired - SparkSession spark; - - @Autowired - FhirContext fhirContext; - - static final String ID_ALIAS = "_abc123"; - - @Test - void comparesDates() { - final Optional optionalLeftDefinition = FhirHelpers - .getChildOfResource(fhirContext, "MedicationRequest", "authoredOn"); - assertTrue(optionalLeftDefinition.isPresent()); - final ElementDefinition leftDefinition = optionalLeftDefinition.get(); - - final Dataset leftDataset = new DatasetBuilder(spark) - .withIdColumn(ID_ALIAS) - .withColumn(DataTypes.StringType) - .withRow("patient-1", "2013-06-10T15:33:22Z") - .withRow("patient-2", "2014-09-25T22:04:19+10:00") - .withRow("patient-3", "2018-05-18T11:03:55-05:00") - .build(); - final ElementPath leftPath = new ElementPathBuilder(spark) - .dataset(leftDataset) - .idAndValueColumns() - .expression("authoredOn") - .singular(true) - .definition(leftDefinition) - .buildDefined(); - - final Optional optionalRightDefinition = FhirHelpers - .getChildOfResource(fhirContext, "Condition", "onsetDateTime"); - assertTrue(optionalRightDefinition.isPresent()); - final ElementDefinition rightDefinition = optionalRightDefinition.get(); - - final Dataset rightDataset = new DatasetBuilder(spark) - .withIdColumn(ID_ALIAS) - .withColumn(DataTypes.StringType) - .withRow("patient-1", "2013-06-10T12:33:22Z") - .withRow("patient-2", "2014-09-25T12:04:19Z") - .withRow("patient-3", "2018-05-19T11:03:55.123Z") - .build(); - final ElementPath rightPath = new ElementPathBuilder(spark) - .dataset(rightDataset) - .idAndValueColumns() - .expression("reverseResolve(Condition.subject).onsetDateTime") - .singular(true) - .definition(rightDefinition) - .buildDefined(); - - final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) - .groupingColumns(Collections.singletonList(leftPath.getIdColumn())) - .build(); - final OperatorInput comparisonInput = new OperatorInput(parserContext, leftPath, - rightPath); - final Operator lessThan = Operator.getInstance("<="); - final FhirPath result = lessThan.invoke(comparisonInput); - - assertTrue(result instanceof BooleanPath); - assertThat((ElementPath) result) - .hasExpression("authoredOn <= reverseResolve(Condition.subject).onsetDateTime") - .isSingular() - .hasFhirType(FHIRDefinedType.BOOLEAN); - - final Dataset expectedDataset = new DatasetBuilder(spark) - .withIdColumn() - .withColumn(DataTypes.BooleanType) - .withRow("patient-1", false) - .withRow("patient-2", true) - .withRow("patient-3", true) - .build(); - assertThat(result) - .selectOrderedResult() - .hasRows(expectedDataset); - } -} \ No newline at end of file diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java new file mode 100644 index 0000000000..8dbed2e188 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java @@ -0,0 +1,201 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.ElementDefinition; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.FhirHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.Collections; +import java.util.Optional; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author John Grimes + */ +@SpringBootTest +@Tag("UnitTest") +class ComparisonOperatorDateTimeTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final String ID_ALIAS = "_abc123"; + private ElementPath left; + private ElementPath right; + private ParserContext parserContext; + + @BeforeEach + void setUp() { + final Optional optionalLeftDefinition = FhirHelpers + .getChildOfResource(fhirContext, "MedicationRequest", "authoredOn"); + assertTrue(optionalLeftDefinition.isPresent()); + final ElementDefinition leftDefinition = optionalLeftDefinition.get(); + + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2013-06-10T15:33:22Z") // Equal, exact + .withRow("patient-2", "2013-06-10T15:33:22Z") // Equal, different time zones + .withRow("patient-3", "2013-06-10T15:33:22+00:00") // Equal, different time zone syntax + .withRow("patient-4", "2013-06-10T15:33:22.000Z") // Equal, different precisions + .withRow("patient-5", "2013-06-10T15:33:21.900Z") // Less than + .withRow("patient-6", "2013-06-11T15:33:22Z") // Greater than + .build(); + left = new ElementPathBuilder(spark) + .dataset(leftDataset) + .idAndValueColumns() + .expression("authoredOn") + .singular(true) + .definition(leftDefinition) + .buildDefined(); + + final Optional optionalRightDefinition = FhirHelpers + .getChildOfResource(fhirContext, "Condition", "onsetDateTime"); + assertTrue(optionalRightDefinition.isPresent()); + final ElementDefinition rightDefinition = optionalRightDefinition.get(); + + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2013-06-10T15:33:22Z") // Equal, exact + .withRow("patient-2", "2013-06-11T01:33:22+10:00") // Equal, different time zones + .withRow("patient-3", "2013-06-10T15:33:22Z") // Equal, different time zone syntax + .withRow("patient-4", "2013-06-10T15:33:22Z") // Equal, different precisions + .withRow("patient-5", "2013-06-10T15:33:22Z") // Less than + .withRow("patient-6", "2013-06-10T15:33:22Z") // Greater than + .build(); + right = new ElementPathBuilder(spark) + .dataset(rightDataset) + .idAndValueColumns() + .expression("reverseResolve(Condition.subject).onsetDateTime") + .singular(true) + .definition(rightDefinition) + .buildDefined(); + + parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + } + + @Test + void equals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // Equal, exact + RowFactory.create("patient-2", true), // Equal, different time zones + RowFactory.create("patient-3", true), // Equal, different time zone syntax + RowFactory.create("patient-4", true), // Equal, different precisions + RowFactory.create("patient-5", false), // Less than + RowFactory.create("patient-6", false) // Greater than + ); + } + + @Test + void notEquals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("!="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // Equal, exact + RowFactory.create("patient-2", false), // Equal, different time zones + RowFactory.create("patient-3", false), // Equal, different time zone syntax + RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-5", true), // Less than + RowFactory.create("patient-6", true) // Greater than + ); + } + + @Test + void lessThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // Equal, exact + RowFactory.create("patient-2", false), // Equal, different time zones + RowFactory.create("patient-3", false), // Equal, different time zone syntax + RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-5", true), // Less than + RowFactory.create("patient-6", false) // Greater than + ); + } + + @Test + void lessThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // Equal, exact + RowFactory.create("patient-2", true), // Equal, different time zones + RowFactory.create("patient-3", true), // Equal, different time zone syntax + RowFactory.create("patient-4", true), // Equal, different precisions + RowFactory.create("patient-5", true), // Less than + RowFactory.create("patient-6", false) // Greater than + ); + } + + @Test + void greaterThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", false), // Equal, exact + RowFactory.create("patient-2", false), // Equal, different time zones + RowFactory.create("patient-3", false), // Equal, different time zone syntax + RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-5", false), // Less than + RowFactory.create("patient-6", true) // Greater than + ); + } + + @Test + void greaterThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", true), // Equal, exact + RowFactory.create("patient-2", true), // Equal, different time zones + RowFactory.create("patient-3", true), // Equal, different time zone syntax + RowFactory.create("patient-4", true), // Equal, different precisions + RowFactory.create("patient-5", false), // Less than + RowFactory.create("patient-6", true) // Greater than + ); + } + +} \ No newline at end of file From 65a25458c8418b0d57748c819e119846c5192ade Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 12 Apr 2022 13:04:40 +1000 Subject: [PATCH 17/83] Add ComparisonOperatorDateTest and ComparisonOperatorTimeTest --- .../operator/ComparisonOperatorDateTest.java | 232 ++++++++++++++++++ .../ComparisonOperatorDateTimeTest.java | 26 +- .../operator/ComparisonOperatorTimeTest.java | 232 ++++++++++++++++++ 3 files changed, 480 insertions(+), 10 deletions(-) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java new file mode 100644 index 0000000000..9bbbe27977 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java @@ -0,0 +1,232 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.ElementDefinition; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.FhirHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.Collections; +import java.util.Optional; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author John Grimes + */ +@SpringBootTest +@Tag("UnitTest") +class ComparisonOperatorDateTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final String ID_ALIAS = "_abc123"; + private ElementPath left; + private ElementPath right; + private ParserContext parserContext; + + @BeforeEach + void setUp() { + final Optional optionalDefinition = FhirHelpers + .getChildOfResource(fhirContext, "Patient", "birthDate"); + assertTrue(optionalDefinition.isPresent()); + final ElementDefinition definition = optionalDefinition.get(); + assertTrue(definition.getFhirType().isPresent()); + assertEquals(FHIRDefinedType.DATE, definition.getFhirType().get()); + + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-01", "2013-06-10") // Equal, years, months and days + .withRow("patient-02", "2013-06") // Equal, years and months + .withRow("patient-03", "2013") // Equal, years + .withRow("patient-04", "2013-06-10") // Different precisions + .withRow("patient-05", "2013-06-10") // Less than, years, months and days + .withRow("patient-06", "2013-06") // Less than, years and months + .withRow("patient-07", "2013") // Less than, years + .withRow("patient-08", "2013-06-10") // Greater than, years, months and days + .withRow("patient-09", "2013-06") // Greater than, years and months + .withRow("patient-10", "2013") // Greater than, years + .build(); + left = new ElementPathBuilder(spark) + .dataset(leftDataset) + .idAndValueColumns() + .expression("birthDate") + .singular(true) + .definition(definition) + .buildDefined(); + + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-01", "2013-06-10") // Equal, years, months and days + .withRow("patient-02", "2013-06") // Equal, years and months + .withRow("patient-03", "2013") // Equal, years + .withRow("patient-04", "2013-06") // Different precisions + .withRow("patient-05", "2013-06-11") // Less than, years, months and days + .withRow("patient-06", "2013-07") // Less than, years and months + .withRow("patient-07", "2014") // Less than, years + .withRow("patient-08", "2013-05-10") // Greater than, years, months and days + .withRow("patient-09", "2012-06") // Greater than, years and months + .withRow("patient-10", "2012") // Greater than, years + .build(); + right = new ElementPathBuilder(spark) + .dataset(rightDataset) + .idAndValueColumns() + .expression("birthDate") + .singular(true) + .definition(definition) + .buildDefined(); + + parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + } + + @Test + void equals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, years, months and days + RowFactory.create("patient-02", true), // Equal, years and months + RowFactory.create("patient-03", true), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, years, months and days + RowFactory.create("patient-06", false), // Less than, years and months + RowFactory.create("patient-07", false), // Less than, years + RowFactory.create("patient-08", false), // Greater than, years, months and days + RowFactory.create("patient-09", false), // Greater than, years and months + RowFactory.create("patient-10", false) // Greater than, years + ); + } + + @Test + void notEquals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("!="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, years, months and days + RowFactory.create("patient-02", false), // Equal, years and months + RowFactory.create("patient-03", false), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, years, months and days + RowFactory.create("patient-06", true), // Less than, years and months + RowFactory.create("patient-07", true), // Less than, years + RowFactory.create("patient-08", true), // Greater than, years, months and days + RowFactory.create("patient-09", true), // Greater than, years and months + RowFactory.create("patient-10", true) // Greater than, years + ); + } + + @Test + void lessThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, years, months and days + RowFactory.create("patient-02", false), // Equal, years and months + RowFactory.create("patient-03", false), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, years, months and days + RowFactory.create("patient-06", true), // Less than, years and months + RowFactory.create("patient-07", true), // Less than, years + RowFactory.create("patient-08", false), // Greater than, years, months and days + RowFactory.create("patient-09", false), // Greater than, years and months + RowFactory.create("patient-10", false) // Greater than, years + ); + } + + @Test + void lessThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, years, months and days + RowFactory.create("patient-02", true), // Equal, years and months + RowFactory.create("patient-03", true), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, years, months and days + RowFactory.create("patient-06", true), // Less than, years and months + RowFactory.create("patient-07", true), // Less than, years + RowFactory.create("patient-08", false), // Greater than, years, months and days + RowFactory.create("patient-09", false), // Greater than, years and months + RowFactory.create("patient-10", false) // Greater than, years + ); + } + + @Test + void greaterThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, years, months and days + RowFactory.create("patient-02", false), // Equal, years and months + RowFactory.create("patient-03", false), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, years, months and days + RowFactory.create("patient-06", false), // Less than, years and months + RowFactory.create("patient-07", false), // Less than, years + RowFactory.create("patient-08", true), // Greater than, years, months and days + RowFactory.create("patient-09", true), // Greater than, years and months + RowFactory.create("patient-10", true) // Greater than, years + ); + } + + @Test + void greaterThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, years, months and days + RowFactory.create("patient-02", true), // Equal, years and months + RowFactory.create("patient-03", true), // Equal, years + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, years, months and days + RowFactory.create("patient-06", false), // Less than, years and months + RowFactory.create("patient-07", false), // Less than, years + RowFactory.create("patient-08", true), // Greater than, years, months and days + RowFactory.create("patient-09", true), // Greater than, years and months + RowFactory.create("patient-10", true) // Greater than, years + ); + } + +} \ No newline at end of file diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java index 8dbed2e188..62e4def992 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java @@ -7,6 +7,7 @@ package au.csiro.pathling.fhirpath.operator; import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import au.csiro.pathling.fhirpath.FhirPath; @@ -25,6 +26,7 @@ import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -55,6 +57,8 @@ void setUp() { .getChildOfResource(fhirContext, "MedicationRequest", "authoredOn"); assertTrue(optionalLeftDefinition.isPresent()); final ElementDefinition leftDefinition = optionalLeftDefinition.get(); + assertTrue(leftDefinition.getFhirType().isPresent()); + assertEquals(FHIRDefinedType.DATETIME, leftDefinition.getFhirType().get()); final Dataset leftDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) @@ -78,6 +82,8 @@ void setUp() { .getChildOfResource(fhirContext, "Condition", "onsetDateTime"); assertTrue(optionalRightDefinition.isPresent()); final ElementDefinition rightDefinition = optionalRightDefinition.get(); + assertTrue(rightDefinition.getFhirType().isPresent()); + assertEquals(FHIRDefinedType.DATETIME, rightDefinition.getFhirType().get()); final Dataset rightDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) @@ -112,7 +118,7 @@ void equals() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", true), // Equal, different precisions + RowFactory.create("patient-4", null), // Equal, different precisions RowFactory.create("patient-5", false), // Less than RowFactory.create("patient-6", false) // Greater than ); @@ -128,7 +134,7 @@ void notEquals() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-4", null), // Equal, different precisions RowFactory.create("patient-5", true), // Less than RowFactory.create("patient-6", true) // Greater than ); @@ -144,7 +150,7 @@ void lessThan() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-4", null), // Equal, different precisions RowFactory.create("patient-5", true), // Less than RowFactory.create("patient-6", false) // Greater than ); @@ -160,9 +166,9 @@ void lessThanOrEqualTo() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", true), // Equal, different precisions - RowFactory.create("patient-5", true), // Less than - RowFactory.create("patient-6", false) // Greater than + RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-5", true), // Less than + RowFactory.create("patient-6", false) // Greater than ); } @@ -176,7 +182,7 @@ void greaterThan() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", false), // Equal, different precisions + RowFactory.create("patient-4", null), // Equal, different precisions RowFactory.create("patient-5", false), // Less than RowFactory.create("patient-6", true) // Greater than ); @@ -192,9 +198,9 @@ void greaterThanOrEqualTo() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", true), // Equal, different precisions - RowFactory.create("patient-5", false), // Less than - RowFactory.create("patient-6", true) // Greater than + RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-5", false), // Less than + RowFactory.create("patient-6", true) // Greater than ); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java new file mode 100644 index 0000000000..045ee29cae --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java @@ -0,0 +1,232 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.ElementDefinition; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.FhirHelpers; +import ca.uhn.fhir.context.FhirContext; +import java.util.Collections; +import java.util.Optional; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author John Grimes + */ +@SpringBootTest +@Tag("UnitTest") +class ComparisonOperatorTimeTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final String ID_ALIAS = "_abc123"; + private ElementPath left; + private ElementPath right; + private ParserContext parserContext; + + @BeforeEach + void setUp() { + final Optional optionalDefinition = FhirHelpers + .getChildOfResource(fhirContext, "Observation", "valueTime"); + assertTrue(optionalDefinition.isPresent()); + final ElementDefinition definition = optionalDefinition.get(); + assertTrue(definition.getFhirType().isPresent()); + assertEquals(FHIRDefinedType.TIME, definition.getFhirType().get()); + + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-01", "03:15:36") // Equal, hours, minutes and seconds + .withRow("patient-02", "03:15") // Equal, hours and minutes + .withRow("patient-03", "03") // Equal, hours + .withRow("patient-04", "03:15:36") // Different precisions + .withRow("patient-05", "03:15:36") // Less than, hours, minutes and seconds + .withRow("patient-06", "03:15") // Less than, hours and minutes + .withRow("patient-07", "03") // Less than, hours + .withRow("patient-08", "03:15:36") // Greater than, hours, minutes and seconds + .withRow("patient-09", "03:15") // Greater than, hours and minutes + .withRow("patient-10", "03") // Greater than, hours + .build(); + left = new ElementPathBuilder(spark) + .dataset(leftDataset) + .idAndValueColumns() + .expression("valueTime") + .singular(true) + .definition(definition) + .buildDefined(); + + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-01", "03:15:36") // Equal, hours, minutes and seconds + .withRow("patient-02", "03:15") // Equal, hours and minutes + .withRow("patient-03", "03") // Equal, hours + .withRow("patient-04", "03:15") // Different precisions + .withRow("patient-05", "03:16:36") // Less than, hours, minutes and seconds + .withRow("patient-06", "03:16") // Less than, hours and minutes + .withRow("patient-07", "04") // Less than, hours + .withRow("patient-08", "02:15:36") // Greater than, hours, minutes and seconds + .withRow("patient-09", "03:14") // Greater than, hours and minutes + .withRow("patient-10", "01") // Greater than, hours + .build(); + right = new ElementPathBuilder(spark) + .dataset(rightDataset) + .idAndValueColumns() + .expression("valueTime") + .singular(true) + .definition(definition) + .buildDefined(); + + parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + } + + @Test + void equals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", true), // Equal, hours and minutes + RowFactory.create("patient-03", true), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", false), // Less than, hours and minutes + RowFactory.create("patient-07", false), // Less than, hours + RowFactory.create("patient-08", false), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", false), // Greater than, hours and minutes + RowFactory.create("patient-10", false) // Greater than, hours + ); + } + + @Test + void notEquals() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("!="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", false), // Equal, hours and minutes + RowFactory.create("patient-03", false), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", true), // Less than, hours and minutes + RowFactory.create("patient-07", true), // Less than, hours + RowFactory.create("patient-08", true), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", true), // Greater than, hours and minutes + RowFactory.create("patient-10", true) // Greater than, hours + ); + } + + @Test + void lessThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", false), // Equal, hours and minutes + RowFactory.create("patient-03", false), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", true), // Less than, hours and minutes + RowFactory.create("patient-07", true), // Less than, hours + RowFactory.create("patient-08", false), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", false), // Greater than, hours and minutes + RowFactory.create("patient-10", false) // Greater than, hours + ); + } + + @Test + void lessThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance("<="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", true), // Equal, hours and minutes + RowFactory.create("patient-03", true), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", true), // Less than, hours and minutes + RowFactory.create("patient-07", true), // Less than, hours + RowFactory.create("patient-08", false), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", false), // Greater than, hours and minutes + RowFactory.create("patient-10", false) // Greater than, hours + ); + } + + @Test + void greaterThan() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">"); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", false), // Equal, hours and minutes + RowFactory.create("patient-03", false), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", false), // Less than, hours and minutes + RowFactory.create("patient-07", false), // Less than, hours + RowFactory.create("patient-08", true), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", true), // Greater than, hours and minutes + RowFactory.create("patient-10", true) // Greater than, hours + ); + } + + @Test + void greaterThanOrEqualTo() { + final OperatorInput comparisonInput = new OperatorInput(parserContext, left, right); + final Operator lessThan = Operator.getInstance(">="); + final FhirPath result = lessThan.invoke(comparisonInput); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", true), // Equal, hours and minutes + RowFactory.create("patient-03", true), // Equal, hours + RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", false), // Less than, hours and minutes + RowFactory.create("patient-07", false), // Less than, hours + RowFactory.create("patient-08", true), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", true), // Greater than, hours and minutes + RowFactory.create("patient-10", true) // Greater than, hours + ); + } + +} \ No newline at end of file From 26022b081f49385677f4141d2a71e4dd0648ada2 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 13 Apr 2022 17:14:32 +1000 Subject: [PATCH 18/83] Add comparison operator tests and refactor comparison implementation --- .../pathling/fhirpath/element/DatePath.java | 8 +-- .../fhirpath/element/DateTimePath.java | 59 ++++++++++++++----- .../pathling/fhirpath/element/TimePath.java | 6 +- .../fhirpath/literal/DateLiteralPath.java | 8 +-- .../fhirpath/literal/DateTimeLiteralPath.java | 8 +-- .../fhirpath/literal/TimeLiteralPath.java | 6 +- .../sql/dates/TemporalArithmeticFunction.java | 6 +- .../sql/dates/TemporalComparisonFunction.java | 41 +++++++++++++ .../DateAddDurationFunction.java} | 14 ++--- .../{ => date}/DateArithmeticFunction.java | 7 ++- .../DateSubtractDurationFunction.java} | 14 ++--- .../DateTimeAddDurationFunction.java} | 14 ++--- .../DateTimeArithmeticFunction.java | 7 ++- .../datetime/DateTimeComparisonFunction.java | 22 +++++++ .../datetime/DateTimeEqualsFunction.java | 33 +++++++++++ .../datetime/DateTimeGreaterThanFunction.java | 35 +++++++++++ .../DateTimeGreaterThanOrEqualToFunction.java | 34 +++++++++++ .../datetime/DateTimeLessThanFunction.java | 35 +++++++++++ .../DateTimeLessThanOrEqualToFunction.java | 34 +++++++++++ .../DateTimeSubtractDurationFunction.java} | 15 ++--- .../TimeAddDurationFunction.java} | 14 ++--- .../{ => time}/TimeArithmeticFunction.java | 15 ++--- .../dates/time/TimeComparisonFunction.java | 22 +++++++ .../sql/dates/time/TimeEqualsFunction.java | 32 ++++++++++ .../pathling/sql/dates/time/TimeFunction.java | 25 ++++++++ .../dates/time/TimeGreaterThanFunction.java | 32 ++++++++++ .../TimeGreaterThanOrEqualToFunction.java | 32 ++++++++++ .../sql/dates/time/TimeLessThanFunction.java | 32 ++++++++++ .../time/TimeLessThanOrEqualToFunction.java | 32 ++++++++++ .../TimeSubtractDurationFunction.java} | 16 ++--- .../operator/ComparisonOperatorDateTest.java | 32 +++++----- .../ComparisonOperatorDateTimeTest.java | 12 ++-- .../operator/ComparisonOperatorTimeTest.java | 30 +++++----- .../pathling/test/UnitTestDependencies.java | 34 ++++++++--- 34 files changed, 625 insertions(+), 141 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{AddDurationToDate.java => date/DateAddDurationFunction.java} (70%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{ => date}/DateArithmeticFunction.java (77%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{SubtractDurationFromDate.java => date/DateSubtractDurationFunction.java} (70%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{AddDurationToDateTime.java => datetime/DateTimeAddDurationFunction.java} (69%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{ => datetime}/DateTimeArithmeticFunction.java (81%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{SubtractDurationFromDateTime.java => datetime/DateTimeSubtractDurationFunction.java} (68%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{AddDurationToTime.java => time/TimeAddDurationFunction.java} (70%) rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{ => time}/TimeArithmeticFunction.java (55%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java rename fhir-server/src/main/java/au/csiro/pathling/sql/dates/{SubtractDurationFromTime.java => time/TimeSubtractDurationFunction.java} (70%) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java index 536b6629b2..212b8f6243 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java @@ -17,8 +17,8 @@ import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; -import au.csiro.pathling.sql.dates.AddDurationToDate; -import au.csiro.pathling.sql.dates.SubtractDurationFromDate; +import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; +import au.csiro.pathling.sql.dates.date.DateSubtractDurationFunction; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -129,7 +129,7 @@ public static Optional valueFromRow(@Nonnull final Row row, final int @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation.getSparkFunction()); + return DateTimePath.buildComparison(this, operation); } @Override @@ -148,7 +148,7 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToDate.FUNCTION_NAME, SubtractDurationFromDate.FUNCTION_NAME); + DateAddDurationFunction.FUNCTION_NAME, DateSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index 789c3a496c..15ba445c4b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -7,7 +7,8 @@ package au.csiro.pathling.fhirpath.element; import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; -import static org.apache.spark.sql.functions.to_timestamp; +import static org.apache.spark.sql.functions.callUDF; +import static org.apache.spark.sql.functions.not; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -19,14 +20,19 @@ import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; -import au.csiro.pathling.sql.dates.AddDurationToDateTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; +import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeEqualsFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.TimeZone; -import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; +import javolution.testing.AssertionException; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -94,18 +100,41 @@ public static Optional valueFromRow(@Nonnull final Row row, /** * Builds a comparison function for date and date/time like paths. * - * @param source The path to build the comparison function for - * @param sparkFunction The Spark column function to use - * @return A new {@link Function} + * @param source the path to build the comparison function for + * @param operation the {@link ComparisonOperation} that should be built + * @return a new {@link Function} */ @Nonnull public static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final BiFunction sparkFunction) { - // The value columns are converted to native Spark timestamps before comparison. The reason that - // we don't use an explicit format string here is that we require flexibility to accommodate the - // optionality of the milliseconds component of the FHIR date time format. - return target -> sparkFunction - .apply(to_timestamp(source.getValueColumn()), to_timestamp(target.getValueColumn())); + @Nonnull final ComparisonOperation operation) { + return (target) -> { + final String functionName; + switch (operation) { + case EQUALS: + case NOT_EQUALS: + functionName = DateTimeEqualsFunction.FUNCTION_NAME; + final Column equals = callUDF(functionName, + source.getValueColumn(), target.getValueColumn()); + return operation == ComparisonOperation.EQUALS + ? equals + : not(equals); + case LESS_THAN: + functionName = DateTimeLessThanFunction.FUNCTION_NAME; + break; + case LESS_THAN_OR_EQUAL_TO: + functionName = DateTimeLessThanOrEqualToFunction.FUNCTION_NAME; + break; + case GREATER_THAN: + functionName = DateTimeGreaterThanFunction.FUNCTION_NAME; + break; + case GREATER_THAN_OR_EQUAL_TO: + functionName = DateTimeGreaterThanOrEqualToFunction.FUNCTION_NAME; + break; + default: + throw new AssertionException("Unsupported operation: " + operation); + } + return callUDF(functionName, source.getValueColumn(), target.getValueColumn()); + }; } public static TimeZone getDefaultTimeZone() { @@ -120,7 +149,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return buildComparison(this, operation.getSparkFunction()); + return buildComparison(this, operation); } @Override @@ -139,7 +168,7 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToDateTime.FUNCTION_NAME, SubtractDurationFromDateTime.FUNCTION_NAME); + DateTimeAddDurationFunction.FUNCTION_NAME, DateTimeSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java index 0dafe04c79..445331d4bb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java @@ -17,8 +17,8 @@ import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; -import au.csiro.pathling.sql.dates.AddDurationToTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromTime; +import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; +import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.Function; @@ -97,6 +97,6 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToTime.FUNCTION_NAME, SubtractDurationFromTime.FUNCTION_NAME); + TimeAddDurationFunction.FUNCTION_NAME, TimeSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java index 3584aafc88..6b508ab3d2 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java @@ -16,8 +16,8 @@ import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.DatePath; import au.csiro.pathling.fhirpath.element.DateTimePath; -import au.csiro.pathling.sql.dates.AddDurationToDate; -import au.csiro.pathling.sql.dates.SubtractDurationFromDate; +import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; +import au.csiro.pathling.sql.dates.date.DateSubtractDurationFunction; import java.text.ParseException; import java.util.Optional; import java.util.function.Function; @@ -76,7 +76,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation.getSparkFunction()); + return DateTimePath.buildComparison(this, operation); } @Override @@ -101,7 +101,7 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToDate.FUNCTION_NAME, SubtractDurationFromDate.FUNCTION_NAME); + DateAddDurationFunction.FUNCTION_NAME, DateSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java index ba1b3bb3f8..0fa1dc1a1a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java @@ -15,8 +15,8 @@ import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.DateTimePath; -import au.csiro.pathling.sql.dates.AddDurationToDateTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; +import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; import java.text.ParseException; import java.util.Optional; import java.util.function.Function; @@ -79,7 +79,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation.getSparkFunction()); + return DateTimePath.buildComparison(this, operation); } @Override @@ -105,7 +105,7 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToDateTime.FUNCTION_NAME, SubtractDurationFromDateTime.FUNCTION_NAME); + DateTimeAddDurationFunction.FUNCTION_NAME, DateTimeSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java index ca028b9d61..a079d7676e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java @@ -15,8 +15,8 @@ import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.TimePath; -import au.csiro.pathling.sql.dates.AddDurationToTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromTime; +import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; +import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; @@ -99,6 +99,6 @@ public Function getDateArithmeticOperation( @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, @Nonnull final String expression) { return buildDateArithmeticOperation(this, operation, dataset, expression, - AddDurationToTime.FUNCTION_NAME, SubtractDurationFromTime.FUNCTION_NAME); + TimeAddDurationFunction.FUNCTION_NAME, TimeSubtractDurationFunction.FUNCTION_NAME); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index 235029f0eb..087bc55c54 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -85,11 +85,11 @@ protected IntermediateType performSubtraction(@Nonnull final IntermediateType te return (IntermediateType) temporal.minus(amountToAdd, temporalUnit); } - abstract Function parseEncodedValue(); + protected abstract Function parseEncodedValue(); - abstract BiFunction getOperationFunction(); + protected abstract BiFunction getOperationFunction(); - abstract Function encodeResult(); + protected abstract Function encodeResult(); @Override public DataType getReturnType() { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java new file mode 100644 index 0000000000..7341b45bb3 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.sql.udf.SqlFunction2; +import java.util.function.BiFunction; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; + +public abstract class TemporalComparisonFunction implements + SqlFunction2 { + + private static final long serialVersionUID = 492467651418666881L; + + protected abstract Function parseEncodedValue(); + + protected abstract BiFunction getOperationFunction(); + + @Override + public DataType getReturnType() { + return DataTypes.BooleanType; + } + + @Nullable + @Override + public Boolean call(@Nullable final String left, @Nullable final String right) throws Exception { + if (left == null || right == null) { + return null; + } + final IntermediateType parsedLeft = parseEncodedValue().apply(left); + final IntermediateType parsedRight = parseEncodedValue().apply(right); + return getOperationFunction().apply(parsedLeft, parsedRight); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java similarity index 70% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java index 7e4a54304e..4130556f2f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDate.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.date; import java.time.LocalDate; import java.util.function.BiFunction; @@ -14,20 +14,20 @@ @Component @Profile("core") -public class AddDurationToDate extends DateArithmeticFunction { +public class DateAddDurationFunction extends DateArithmeticFunction { private static final long serialVersionUID = -5029179160644275584L; - public static final String FUNCTION_NAME = "add_duration_to_date"; + public static final String FUNCTION_NAME = "date_add_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performAddition; } @Override - BiFunction getOperationFunction() { - return this::performAddition; + public String getName() { + return FUNCTION_NAME; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java similarity index 77% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java index d890de21ea..b9d60993b8 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java @@ -4,8 +4,9 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.date; +import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneOffset; @@ -18,12 +19,12 @@ public abstract class DateArithmeticFunction extends TemporalArithmeticFunction< private static final long serialVersionUID = 6759548804191034570L; @Override - Function parseEncodedValue() { + protected Function parseEncodedValue() { return LocalDate::parse; } @Override - Function encodeResult() { + protected Function encodeResult() { return (resultDate) -> new DateType( new Date(resultDate.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) * 1000)) .getValueAsString(); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java similarity index 70% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java index 65ad41b160..5e1e96064f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDate.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.date; import java.time.LocalDate; import java.util.function.BiFunction; @@ -14,20 +14,20 @@ @Component @Profile("core") -public class SubtractDurationFromDate extends DateArithmeticFunction { +public class DateSubtractDurationFunction extends DateArithmeticFunction { private static final long serialVersionUID = 5201879133976866457L; - public static final String FUNCTION_NAME = "subtract_duration_from_date"; + public static final String FUNCTION_NAME = "date_subtract_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performSubtraction; } @Override - BiFunction getOperationFunction() { - return this::performSubtraction; + public String getName() { + return FUNCTION_NAME; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java similarity index 69% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java index 3f09e1d3bd..10a3fb32ab 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToDateTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.datetime; import java.time.ZonedDateTime; import java.util.function.BiFunction; @@ -14,20 +14,20 @@ @Component @Profile("core") -public class AddDurationToDateTime extends DateTimeArithmeticFunction { +public class DateTimeAddDurationFunction extends DateTimeArithmeticFunction { private static final long serialVersionUID = 6922227603585641053L; - public static final String FUNCTION_NAME = "add_duration_to_datetime"; + public static final String FUNCTION_NAME = "datetime_add_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performAddition; } @Override - BiFunction getOperationFunction() { - return this::performAddition; + public String getName() { + return FUNCTION_NAME; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java similarity index 81% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java index 77183ae688..ba7a88b8ee 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/DateTimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java @@ -4,9 +4,10 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.datetime; import au.csiro.pathling.fhirpath.element.DateTimePath; +import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import java.time.ZonedDateTime; import java.util.Date; @@ -19,12 +20,12 @@ public abstract class DateTimeArithmeticFunction extends TemporalArithmeticFunct private static final long serialVersionUID = -6669722492626320119L; @Override - Function parseEncodedValue() { + protected Function parseEncodedValue() { return ZonedDateTime::parse; } @Override - Function encodeResult() { + protected Function encodeResult() { return (resultDateTime) -> { final BaseDateTimeType dateTime = new DateTimeType(Date.from(resultDateTime.toInstant())) .setTimeZone(DateTimePath.getDefaultTimeZone()); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java new file mode 100644 index 0000000000..e6b0cc10dd --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import au.csiro.pathling.sql.dates.TemporalComparisonFunction; +import java.util.function.Function; +import org.hl7.fhir.r4.model.DateTimeType; + +public abstract class DateTimeComparisonFunction extends TemporalComparisonFunction { + + private static final long serialVersionUID = -2449192480093120211L; + + @Override + protected Function parseEncodedValue() { + return DateTimeType::new; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java new file mode 100644 index 0000000000..2667c8689a --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.BaseDateTimeType; +import org.hl7.fhir.r4.model.DateTimeType; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class DateTimeEqualsFunction extends DateTimeComparisonFunction { + + private static final long serialVersionUID = -8717420985056046161L; + + public static final String FUNCTION_NAME = "datetime_eq"; + + @Override + protected BiFunction getOperationFunction() { + return BaseDateTimeType::equalsUsingFhirPathRules; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java new file mode 100644 index 0000000000..3ea5108b4d --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import java.util.function.BiFunction; +import javax.annotation.Nonnull; +import org.hl7.fhir.r4.model.BaseDateTimeType; +import org.hl7.fhir.r4.model.DateTimeType; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class DateTimeGreaterThanFunction extends DateTimeComparisonFunction { + + private static final long serialVersionUID = 6648102436817402989L; + + public static final String FUNCTION_NAME = "datetime_gt"; + + @Nonnull + @Override + protected BiFunction getOperationFunction() { + return BaseDateTimeType::after; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java new file mode 100644 index 0000000000..bc554e5de7 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import java.util.function.BiFunction; +import javax.annotation.Nonnull; +import org.hl7.fhir.r4.model.DateTimeType; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class DateTimeGreaterThanOrEqualToFunction extends DateTimeComparisonFunction { + + private static final long serialVersionUID = -4680194983727139029L; + + public static final String FUNCTION_NAME = "datetime_gte"; + + @Nonnull + @Override + protected BiFunction getOperationFunction() { + return (left, right) -> left.after(right) || left.equalsUsingFhirPathRules(right); + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java new file mode 100644 index 0000000000..d39354380f --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import java.util.function.BiFunction; +import javax.annotation.Nonnull; +import org.hl7.fhir.r4.model.BaseDateTimeType; +import org.hl7.fhir.r4.model.DateTimeType; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class DateTimeLessThanFunction extends DateTimeComparisonFunction { + + private static final long serialVersionUID = -4688679934965980217L; + + public static final String FUNCTION_NAME = "datetime_lt"; + + @Nonnull + @Override + protected BiFunction getOperationFunction() { + return BaseDateTimeType::before; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java new file mode 100644 index 0000000000..449bf2e2d1 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.datetime; + +import java.util.function.BiFunction; +import javax.annotation.Nonnull; +import org.hl7.fhir.r4.model.DateTimeType; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class DateTimeLessThanOrEqualToFunction extends DateTimeComparisonFunction { + + private static final long serialVersionUID = 787654631927909813L; + + public static final String FUNCTION_NAME = "datetime_lte"; + + @Nonnull + @Override + protected BiFunction getOperationFunction() { + return (left, right) -> left.before(right) || left.equalsUsingFhirPathRules(right); + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java similarity index 68% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java index bd0463bdf1..291342e9f1 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromDateTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.datetime; import java.time.ZonedDateTime; import java.util.function.BiFunction; @@ -14,20 +14,21 @@ @Component @Profile("core") -public class SubtractDurationFromDateTime extends DateTimeArithmeticFunction { +public class DateTimeSubtractDurationFunction extends DateTimeArithmeticFunction { private static final long serialVersionUID = -5922228168177608861L; - public static final String FUNCTION_NAME = "subtract_duration_from_datetime"; + public static final String FUNCTION_NAME = "datetime_subtract_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performSubtraction; } @Override - BiFunction getOperationFunction() { - return this::performSubtraction; + public String getName() { + return FUNCTION_NAME; } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java similarity index 70% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java index e889c2ee80..86d8f7c898 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/AddDurationToTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.time; import java.time.LocalTime; import java.util.function.BiFunction; @@ -14,20 +14,20 @@ @Component @Profile("core") -public class AddDurationToTime extends TimeArithmeticFunction { +public class TimeAddDurationFunction extends TimeArithmeticFunction { private static final long serialVersionUID = 5839806160423512490L; - public static final String FUNCTION_NAME = "add_duration_to_time"; + public static final String FUNCTION_NAME = "time_add_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performAddition; } @Override - BiFunction getOperationFunction() { - return this::performAddition; + public String getName() { + return FUNCTION_NAME; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java similarity index 55% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java index be1c3c7cbb..16aba024bc 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java @@ -4,29 +4,24 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.time; +import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; import java.time.LocalTime; import java.util.function.Function; -import java.util.regex.Pattern; import org.hl7.fhir.r4.model.TimeType; public abstract class TimeArithmeticFunction extends TemporalArithmeticFunction { private static final long serialVersionUID = 7504107794184508523L; - private static final Pattern HOURS_ONLY = Pattern.compile("^\\d{2}$"); - @Override - Function parseEncodedValue() { - // LocalDate will not successfully parse the HH format. - return (temporal) -> HOURS_ONLY.matcher(temporal).matches() - ? LocalTime.parse(temporal + ":00") - : LocalTime.parse(temporal); + protected Function parseEncodedValue() { + return TimeFunction::parseEncodedTime; } @Override - Function encodeResult() { + protected Function encodeResult() { return (resultTime) -> new TimeType(resultTime.toString()) .getValueAsString(); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java new file mode 100644 index 0000000000..c64098e779 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import au.csiro.pathling.sql.dates.TemporalComparisonFunction; +import java.time.LocalTime; +import java.util.function.Function; + +public abstract class TimeComparisonFunction extends TemporalComparisonFunction { + + private static final long serialVersionUID = 3661335567427062952L; + + @Override + protected Function parseEncodedValue() { + return TimeFunction::parseEncodedTime; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java new file mode 100644 index 0000000000..9c430205dd --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TimeEqualsFunction extends TimeComparisonFunction { + + private static final long serialVersionUID = -6607019777915271539L; + + public static final String FUNCTION_NAME = "time_eq"; + + @Override + protected BiFunction getOperationFunction() { + return LocalTime::equals; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java new file mode 100644 index 0000000000..39198bafdb --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.regex.Pattern; +import javax.annotation.Nonnull; + +abstract class TimeFunction { + + private static final Pattern HOURS_ONLY = Pattern.compile("^\\d{2}$"); + + @Nonnull + static LocalTime parseEncodedTime(@Nonnull final String encoded) { + // LocalDate will not successfully parse the HH format. + return HOURS_ONLY.matcher(encoded).matches() + ? LocalTime.parse(encoded + ":00") + : LocalTime.parse(encoded); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java new file mode 100644 index 0000000000..edd6b033db --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TimeGreaterThanFunction extends TimeComparisonFunction { + + private static final long serialVersionUID = 1785863110270729355L; + + public static final String FUNCTION_NAME = "time_gt"; + + @Override + protected BiFunction getOperationFunction() { + return LocalTime::isAfter; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java new file mode 100644 index 0000000000..e70ccb6a58 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TimeGreaterThanOrEqualToFunction extends TimeComparisonFunction { + + private static final long serialVersionUID = -8098751595303018323L; + + public static final String FUNCTION_NAME = "time_gte"; + + @Override + protected BiFunction getOperationFunction() { + return (a, b) -> a.isAfter(b) || a.equals(b); + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java new file mode 100644 index 0000000000..29d1d46802 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TimeLessThanFunction extends TimeComparisonFunction { + + private static final long serialVersionUID = 5957315745114053261L; + + public static final String FUNCTION_NAME = "time_lt"; + + @Override + protected BiFunction getOperationFunction() { + return LocalTime::isBefore; + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java new file mode 100644 index 0000000000..40890f5bfd --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates.time; + +import java.time.LocalTime; +import java.util.function.BiFunction; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TimeLessThanOrEqualToFunction extends TimeComparisonFunction { + + private static final long serialVersionUID = -5929640258789711609L; + + public static final String FUNCTION_NAME = "time_lte"; + + @Override + protected BiFunction getOperationFunction() { + return (a, b) -> a.isBefore(b) || a.equals(b); + } + + @Override + public String getName() { + return FUNCTION_NAME; + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java similarity index 70% rename from fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java rename to fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java index 7dcaa51f98..2dc5fc5e1c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/SubtractDurationFromTime.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java @@ -4,7 +4,7 @@ * Software Licence Agreement. */ -package au.csiro.pathling.sql.dates; +package au.csiro.pathling.sql.dates.time; import java.time.LocalTime; import java.util.function.BiFunction; @@ -14,20 +14,20 @@ @Component @Profile("core") -public class SubtractDurationFromTime extends TimeArithmeticFunction { +public class TimeSubtractDurationFunction extends TimeArithmeticFunction { private static final long serialVersionUID = 1909732257090337898L; - - public static final String FUNCTION_NAME = "subtract_duration_from_time"; + + public static final String FUNCTION_NAME = "time_subtract_duration"; @Override - public String getName() { - return FUNCTION_NAME; + protected BiFunction getOperationFunction() { + return this::performSubtraction; } @Override - BiFunction getOperationFunction() { - return this::performSubtraction; + public String getName() { + return FUNCTION_NAME; } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java index 9bbbe27977..ed0bd079a4 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTest.java @@ -66,7 +66,7 @@ void setUp() { .withRow("patient-01", "2013-06-10") // Equal, years, months and days .withRow("patient-02", "2013-06") // Equal, years and months .withRow("patient-03", "2013") // Equal, years - .withRow("patient-04", "2013-06-10") // Different precisions + .withRow("patient-04", "2013-06-01") // Different precisions .withRow("patient-05", "2013-06-10") // Less than, years, months and days .withRow("patient-06", "2013-06") // Less than, years and months .withRow("patient-07", "2013") // Less than, years @@ -116,16 +116,16 @@ void equals() { final FhirPath result = lessThan.invoke(comparisonInput); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-01", true), // Equal, years, months and days - RowFactory.create("patient-02", true), // Equal, years and months - RowFactory.create("patient-03", true), // Equal, years - RowFactory.create("patient-04", null), // Different precisions - RowFactory.create("patient-05", false), // Less than, years, months and days - RowFactory.create("patient-06", false), // Less than, years and months - RowFactory.create("patient-07", false), // Less than, years - RowFactory.create("patient-08", false), // Greater than, years, months and days - RowFactory.create("patient-09", false), // Greater than, years and months - RowFactory.create("patient-10", false) // Greater than, years + RowFactory.create("patient-01", true), // Equal, years, months and days + RowFactory.create("patient-02", true), // Equal, years and months + RowFactory.create("patient-03", true), // Equal, years + RowFactory.create("patient-04", true), // Different precisions + RowFactory.create("patient-05", false), // Less than, years, months and days + RowFactory.create("patient-06", false), // Less than, years and months + RowFactory.create("patient-07", false), // Less than, years + RowFactory.create("patient-08", false), // Greater than, years, months and days + RowFactory.create("patient-09", false), // Greater than, years and months + RowFactory.create("patient-10", false) // Greater than, years ); } @@ -139,7 +139,7 @@ void notEquals() { RowFactory.create("patient-01", false), // Equal, years, months and days RowFactory.create("patient-02", false), // Equal, years and months RowFactory.create("patient-03", false), // Equal, years - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", false), // Different precisions RowFactory.create("patient-05", true), // Less than, years, months and days RowFactory.create("patient-06", true), // Less than, years and months RowFactory.create("patient-07", true), // Less than, years @@ -159,7 +159,7 @@ void lessThan() { RowFactory.create("patient-01", false), // Equal, years, months and days RowFactory.create("patient-02", false), // Equal, years and months RowFactory.create("patient-03", false), // Equal, years - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", false), // Different precisions RowFactory.create("patient-05", true), // Less than, years, months and days RowFactory.create("patient-06", true), // Less than, years and months RowFactory.create("patient-07", true), // Less than, years @@ -179,7 +179,7 @@ void lessThanOrEqualTo() { RowFactory.create("patient-01", true), // Equal, years, months and days RowFactory.create("patient-02", true), // Equal, years and months RowFactory.create("patient-03", true), // Equal, years - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", true), // Different precisions RowFactory.create("patient-05", true), // Less than, years, months and days RowFactory.create("patient-06", true), // Less than, years and months RowFactory.create("patient-07", true), // Less than, years @@ -199,7 +199,7 @@ void greaterThan() { RowFactory.create("patient-01", false), // Equal, years, months and days RowFactory.create("patient-02", false), // Equal, years and months RowFactory.create("patient-03", false), // Equal, years - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", false), // Different precisions RowFactory.create("patient-05", false), // Less than, years, months and days RowFactory.create("patient-06", false), // Less than, years and months RowFactory.create("patient-07", false), // Less than, years @@ -219,7 +219,7 @@ void greaterThanOrEqualTo() { RowFactory.create("patient-01", true), // Equal, years, months and days RowFactory.create("patient-02", true), // Equal, years and months RowFactory.create("patient-03", true), // Equal, years - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", true), // Different precisions RowFactory.create("patient-05", false), // Less than, years, months and days RowFactory.create("patient-06", false), // Less than, years and months RowFactory.create("patient-07", false), // Less than, years diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java index 62e4def992..c92fd4d372 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorDateTimeTest.java @@ -118,7 +118,7 @@ void equals() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", true), // Equal, different precisions RowFactory.create("patient-5", false), // Less than RowFactory.create("patient-6", false) // Greater than ); @@ -134,7 +134,7 @@ void notEquals() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", false), // Equal, different precisions RowFactory.create("patient-5", true), // Less than RowFactory.create("patient-6", true) // Greater than ); @@ -150,7 +150,7 @@ void lessThan() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", false), // Equal, different precisions RowFactory.create("patient-5", true), // Less than RowFactory.create("patient-6", false) // Greater than ); @@ -166,7 +166,7 @@ void lessThanOrEqualTo() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", true), // Equal, different precisions RowFactory.create("patient-5", true), // Less than RowFactory.create("patient-6", false) // Greater than ); @@ -182,7 +182,7 @@ void greaterThan() { RowFactory.create("patient-1", false), // Equal, exact RowFactory.create("patient-2", false), // Equal, different time zones RowFactory.create("patient-3", false), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", false), // Equal, different precisions RowFactory.create("patient-5", false), // Less than RowFactory.create("patient-6", true) // Greater than ); @@ -198,7 +198,7 @@ void greaterThanOrEqualTo() { RowFactory.create("patient-1", true), // Equal, exact RowFactory.create("patient-2", true), // Equal, different time zones RowFactory.create("patient-3", true), // Equal, different time zone syntax - RowFactory.create("patient-4", null), // Equal, different precisions + RowFactory.create("patient-4", true), // Equal, different precisions RowFactory.create("patient-5", false), // Less than RowFactory.create("patient-6", true) // Greater than ); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java index 045ee29cae..8ac0731a04 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/ComparisonOperatorTimeTest.java @@ -116,16 +116,16 @@ void equals() { final FhirPath result = lessThan.invoke(comparisonInput); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds - RowFactory.create("patient-02", true), // Equal, hours and minutes - RowFactory.create("patient-03", true), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions - RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds - RowFactory.create("patient-06", false), // Less than, hours and minutes - RowFactory.create("patient-07", false), // Less than, hours - RowFactory.create("patient-08", false), // Greater than, hours, minutes and seconds - RowFactory.create("patient-09", false), // Greater than, hours and minutes - RowFactory.create("patient-10", false) // Greater than, hours + RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds + RowFactory.create("patient-02", true), // Equal, hours and minutes + RowFactory.create("patient-03", true), // Equal, hours + RowFactory.create("patient-04", false), // Different precisions + RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds + RowFactory.create("patient-06", false), // Less than, hours and minutes + RowFactory.create("patient-07", false), // Less than, hours + RowFactory.create("patient-08", false), // Greater than, hours, minutes and seconds + RowFactory.create("patient-09", false), // Greater than, hours and minutes + RowFactory.create("patient-10", false) // Greater than, hours ); } @@ -139,7 +139,7 @@ void notEquals() { RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds RowFactory.create("patient-02", false), // Equal, hours and minutes RowFactory.create("patient-03", false), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", true), // Different precisions RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds RowFactory.create("patient-06", true), // Less than, hours and minutes RowFactory.create("patient-07", true), // Less than, hours @@ -159,7 +159,7 @@ void lessThan() { RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds RowFactory.create("patient-02", false), // Equal, hours and minutes RowFactory.create("patient-03", false), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", false), // Different precisions RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds RowFactory.create("patient-06", true), // Less than, hours and minutes RowFactory.create("patient-07", true), // Less than, hours @@ -179,7 +179,7 @@ void lessThanOrEqualTo() { RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds RowFactory.create("patient-02", true), // Equal, hours and minutes RowFactory.create("patient-03", true), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", false), // Different precisions RowFactory.create("patient-05", true), // Less than, hours, minutes and seconds RowFactory.create("patient-06", true), // Less than, hours and minutes RowFactory.create("patient-07", true), // Less than, hours @@ -199,7 +199,7 @@ void greaterThan() { RowFactory.create("patient-01", false), // Equal, hours, minutes and seconds RowFactory.create("patient-02", false), // Equal, hours and minutes RowFactory.create("patient-03", false), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", true), // Different precisions RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds RowFactory.create("patient-06", false), // Less than, hours and minutes RowFactory.create("patient-07", false), // Less than, hours @@ -219,7 +219,7 @@ void greaterThanOrEqualTo() { RowFactory.create("patient-01", true), // Equal, hours, minutes and seconds RowFactory.create("patient-02", true), // Equal, hours and minutes RowFactory.create("patient-03", true), // Equal, hours - RowFactory.create("patient-04", null), // Different precisions + RowFactory.create("patient-04", true), // Different precisions RowFactory.create("patient-05", false), // Less than, hours, minutes and seconds RowFactory.create("patient-06", false), // Less than, hours and minutes RowFactory.create("patient-07", false), // Less than, hours diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index cc3731a310..7b6996f0d8 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -12,12 +12,22 @@ import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; -import au.csiro.pathling.sql.dates.AddDurationToDate; -import au.csiro.pathling.sql.dates.AddDurationToDateTime; -import au.csiro.pathling.sql.dates.AddDurationToTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromDate; -import au.csiro.pathling.sql.dates.SubtractDurationFromDateTime; -import au.csiro.pathling.sql.dates.SubtractDurationFromTime; +import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; +import au.csiro.pathling.sql.dates.date.DateSubtractDurationFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeEqualsFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; +import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; +import au.csiro.pathling.sql.dates.time.TimeEqualsFunction; +import au.csiro.pathling.sql.dates.time.TimeGreaterThanFunction; +import au.csiro.pathling.sql.dates.time.TimeGreaterThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.time.TimeLessThanFunction; +import au.csiro.pathling.sql.dates.time.TimeLessThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; import au.csiro.pathling.terminology.CodingToLiteral; @@ -55,9 +65,15 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final UcumService ucumService) { final List sqlFunction1 = List.of(new CodingToLiteral(), new ComparableQuantity(ucumService)); - final List sqlFunction2 = List.of(new AddDurationToDateTime(), - new SubtractDurationFromDateTime(), new AddDurationToDate(), new SubtractDurationFromDate(), - new AddDurationToTime(), new SubtractDurationFromTime()); + final List sqlFunction2 = List.of(new DateTimeAddDurationFunction(), + new DateTimeSubtractDurationFunction(), new DateAddDurationFunction(), + new DateSubtractDurationFunction(), + new TimeAddDurationFunction(), new TimeSubtractDurationFunction(), + new DateTimeEqualsFunction(), new DateTimeGreaterThanFunction(), + new DateTimeGreaterThanOrEqualToFunction(), new DateTimeLessThanFunction(), + new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), + new TimeGreaterThanFunction(), new TimeGreaterThanOrEqualToFunction(), + new TimeLessThanFunction(), new TimeLessThanOrEqualToFunction()); return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2); } From e4336e89102c51ee328bf57346ed40d24f5dd9f4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 14 Apr 2022 00:24:25 +1000 Subject: [PATCH 19/83] Make date arithmetic implementation tolerant of no time zone --- .../datetime/DateTimeAddDurationFunction.java | 4 +- .../datetime/DateTimeArithmeticFunction.java | 61 ++++++++++++++++--- .../DateTimeSubtractDurationFunction.java | 4 +- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java index 10a3fb32ab..024a15ac10 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java @@ -6,7 +6,7 @@ package au.csiro.pathling.sql.dates.datetime; -import java.time.ZonedDateTime; +import java.time.temporal.Temporal; import java.util.function.BiFunction; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; @@ -21,7 +21,7 @@ public class DateTimeAddDurationFunction extends DateTimeArithmeticFunction { public static final String FUNCTION_NAME = "datetime_add_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performAddition; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java index ba7a88b8ee..81d59f0ff7 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java @@ -6,30 +6,73 @@ package au.csiro.pathling.sql.dates.datetime; -import au.csiro.pathling.fhirpath.element.DateTimePath; +import static au.csiro.pathling.utilities.Preconditions.checkNotNull; + import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; import ca.uhn.fhir.model.api.TemporalPrecisionEnum; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.time.temporal.Temporal; import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import java.util.function.Function; import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.DateTimeType; -public abstract class DateTimeArithmeticFunction extends TemporalArithmeticFunction { +public abstract class DateTimeArithmeticFunction extends + TemporalArithmeticFunction { private static final long serialVersionUID = -6669722492626320119L; + static final Map JAVA_TO_HAPI_TIME_UNIT = new ImmutableMap.Builder() + .put(ChronoUnit.MINUTES, TemporalPrecisionEnum.MINUTE) + .put(ChronoUnit.SECONDS, TemporalPrecisionEnum.SECOND) + .put(ChronoUnit.MILLIS, TemporalPrecisionEnum.MILLI) + .build(); + @Override - protected Function parseEncodedValue() { - return ZonedDateTime::parse; + protected Function parseEncodedValue() { + return (encoded) -> { + final DateTimeFormatter formatter = DateTimeFormatter.ofPattern( + "yyyy-MM-dd'T'HH[:mm[:ss[.SSS][XXX]]"); + return (Temporal) formatter.parseBest(encoded, ZonedDateTime::from, LocalDateTime::from); + }; } @Override - protected Function encodeResult() { - return (resultDateTime) -> { - final BaseDateTimeType dateTime = new DateTimeType(Date.from(resultDateTime.toInstant())) - .setTimeZone(DateTimePath.getDefaultTimeZone()); - dateTime.setPrecision(TemporalPrecisionEnum.MILLI); + protected Function encodeResult() { + return (temporal) -> { + final Instant instant = temporal instanceof ZonedDateTime + ? ((ZonedDateTime) temporal).toInstant() + : ((LocalDateTime) temporal).toInstant(ZoneOffset.UTC); + final BaseDateTimeType dateTime = new DateTimeType(Date.from(instant)); + + // Attempt to get the time zone from the parsed object. + try { + final TimeZone timeZone = TimeZone.getTimeZone(temporal.query(ZoneId::from)); + dateTime.setTimeZone(timeZone); + } catch (final Exception ignored) { + } + + // Find the appropriate precision for the result. + for (final ChronoUnit unit : List.of(ChronoUnit.MILLIS, ChronoUnit.SECONDS, + ChronoUnit.MINUTES)) { + if (temporal.isSupported(unit)) { + final TemporalPrecisionEnum precision = JAVA_TO_HAPI_TIME_UNIT.get(unit); + checkNotNull(precision); + dateTime.setPrecision(precision); + break; + } + } + return dateTime.getValueAsString(); }; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java index 291342e9f1..6f7be61ab7 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java @@ -6,7 +6,7 @@ package au.csiro.pathling.sql.dates.datetime; -import java.time.ZonedDateTime; +import java.time.temporal.Temporal; import java.util.function.BiFunction; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; @@ -21,7 +21,7 @@ public class DateTimeSubtractDurationFunction extends DateTimeArithmeticFunction public static final String FUNCTION_NAME = "datetime_subtract_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performSubtraction; } From 9b9d234c3f704195ed88edf0fb0286cd967726ee Mon Sep 17 00:00:00 2001 From: John Grimes Date: Thu, 14 Apr 2022 08:30:06 +1000 Subject: [PATCH 20/83] Align arithmetic implementation to HAPI functionality --- fhir-server/pom.xml | 2 +- .../pathling/fhirpath/element/DatePath.java | 44 +- .../fhirpath/element/DateTimePath.java | 10 - .../pathling/fhirpath/element/TimePath.java | 18 +- .../fhirpath/literal/TimeLiteralPath.java | 15 +- .../sql/dates/TemporalArithmeticFunction.java | 79 ++-- .../dates/date/DateAddDurationFunction.java | 4 +- .../dates/date/DateArithmeticFunction.java | 16 +- .../date/DateSubtractDurationFunction.java | 4 +- .../datetime/DateTimeAddDurationFunction.java | 4 +- .../datetime/DateTimeArithmeticFunction.java | 62 +-- .../DateTimeSubtractDurationFunction.java | 4 +- .../dates/time/TimeAddDurationFunction.java | 33 -- .../dates/time/TimeArithmeticFunction.java | 29 -- .../time/TimeSubtractDurationFunction.java | 33 -- .../fhirpath/operator/DateArithmeticTest.java | 418 ++++-------------- .../pathling/test/UnitTestDependencies.java | 3 - site/docs/fhirpath/data-types.md | 57 +-- site/docs/fhirpath/operators.md | 16 +- 19 files changed, 186 insertions(+), 665 deletions(-) delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 6e91f6ae06..b1082fa5d9 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -343,7 +343,7 @@ ${project.version} - -Xmx2g -XX:MaxMetaspaceSize=400m -XX:ReservedCodeCacheSize=240m -Xss1m + -Xmx2g -XX:MaxMetaspaceSize=400m -XX:ReservedCodeCacheSize=240m -Xss1m -Duser.timezone=UTC diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java index 212b8f6243..999dc8b1dc 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java @@ -19,9 +19,6 @@ import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; import au.csiro.pathling.sql.dates.date.DateSubtractDurationFunction; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; @@ -42,25 +39,6 @@ public class DatePath extends ElementPath implements Materializable, Comparable, Temporal { - private static final ThreadLocal FULL_DATE_FORMAT = ThreadLocal - .withInitial(() -> { - final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - format.setTimeZone(DateTimePath.getDefaultTimeZone()); - return format; - }); - private static final ThreadLocal YEAR_MONTH_DATE_FORMAT = ThreadLocal - .withInitial(() -> { - final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM"); - format.setTimeZone(DateTimePath.getDefaultTimeZone()); - return format; - }); - private static final ThreadLocal YEAR_ONLY_DATE_FORMAT = ThreadLocal - .withInitial(() -> { - final SimpleDateFormat format = new SimpleDateFormat("yyyy"); - format.setTimeZone(DateTimePath.getDefaultTimeZone()); - return format; - }); - protected DatePath(@Nonnull final String expression, @Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Optional eidColumn, @Nonnull final Column valueColumn, final boolean singular, @@ -86,18 +64,6 @@ public static Function buildComparison(@Nonnull final Compar to_timestamp(target.getValueColumn())); } - public static SimpleDateFormat getFullDateFormat() { - return FULL_DATE_FORMAT.get(); - } - - public static SimpleDateFormat getYearMonthDateFormat() { - return YEAR_MONTH_DATE_FORMAT.get(); - } - - public static SimpleDateFormat getYearOnlyDateFormat() { - return YEAR_ONLY_DATE_FORMAT.get(); - } - @Nonnull @Override public Optional getValueFromRow(@Nonnull final Row row, final int columnNumber) { @@ -116,14 +82,8 @@ public static Optional valueFromRow(@Nonnull final Row row, final int if (row.isNullAt(columnNumber)) { return Optional.empty(); } - final Date date; - try { - date = getFullDateFormat().parse(row.getString(columnNumber)); - } catch (final ParseException e) { - log.warn("Error parsing date extracted from row", e); - return Optional.empty(); - } - return Optional.of(new DateType(date)); + final String dateString = row.getString(columnNumber); + return Optional.of(new DateType(dateString)); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index 15ba445c4b..de42862b5c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -29,7 +29,6 @@ import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; import com.google.common.collect.ImmutableSet; import java.util.Optional; -import java.util.TimeZone; import java.util.function.Function; import javax.annotation.Nonnull; import javolution.testing.AssertionException; @@ -49,8 +48,6 @@ public class DateTimePath extends ElementPath implements Materializable, Comparable, Temporal { - private static final TimeZone DEFAULT_TIME_ZONE = TimeZone.getTimeZone("GMT"); - private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(DatePath.class, DateTimePath.class, DateLiteralPath.class, DateTimeLiteralPath.class, NullLiteralPath.class); @@ -85,14 +82,11 @@ public static Optional valueFromRow(@Nonnull final Row row, if (row.isNullAt(columnNumber)) { return Optional.empty(); } - if (fhirType == FHIRDefinedType.INSTANT) { final InstantType value = new InstantType(row.getTimestamp(columnNumber)); - value.setTimeZone(DEFAULT_TIME_ZONE); return Optional.of(value); } else { final DateTimeType value = new DateTimeType(row.getString(columnNumber)); - value.setTimeZone(DEFAULT_TIME_ZONE); return Optional.of(value); } } @@ -137,10 +131,6 @@ public static Function buildComparison(@Nonnull final Compar }; } - public static TimeZone getDefaultTimeZone() { - return DEFAULT_TIME_ZONE; - } - @Nonnull public static ImmutableSet> getComparableTypes() { return COMPARABLE_TYPES; diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java index 445331d4bb..c2f4e25952 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java @@ -6,19 +6,12 @@ package au.csiro.pathling.fhirpath.element; -import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; - import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; -import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; -import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.literal.TimeLiteralPath; -import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; -import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.Function; @@ -34,8 +27,7 @@ * * @author John Grimes */ -public class TimePath extends ElementPath implements Materializable, Comparable, - Temporal { +public class TimePath extends ElementPath implements Materializable, Comparable { private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(TimePath.class, TimeLiteralPath.class, NullLiteralPath.class); @@ -91,12 +83,4 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof TimeLiteralPath; } - @Nonnull - @Override - public Function getDateArithmeticOperation( - @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, - @Nonnull final String expression) { - return buildDateArithmeticOperation(this, operation, dataset, expression, - TimeAddDurationFunction.FUNCTION_NAME, TimeSubtractDurationFunction.FUNCTION_NAME); - } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java index a079d7676e..e4a77b18f9 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java @@ -6,17 +6,12 @@ package au.csiro.pathling.fhirpath.literal; -import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static org.apache.spark.sql.functions.lit; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; -import au.csiro.pathling.fhirpath.Numeric.MathOperation; -import au.csiro.pathling.fhirpath.Temporal; import au.csiro.pathling.fhirpath.element.TimePath; -import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; -import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; @@ -31,7 +26,7 @@ * @author John Grimes */ public class TimeLiteralPath extends LiteralPath implements Materializable, - Comparable, Temporal { + Comparable { protected TimeLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final TimeType literalValue) { @@ -93,12 +88,4 @@ public boolean canBeCombinedWith(@Nonnull final FhirPath target) { return super.canBeCombinedWith(target) || target instanceof TimePath; } - @Nonnull - @Override - public Function getDateArithmeticOperation( - @Nonnull final MathOperation operation, @Nonnull final Dataset dataset, - @Nonnull final String expression) { - return buildDateArithmeticOperation(this, operation, dataset, expression, - TimeAddDurationFunction.FUNCTION_NAME, TimeSubtractDurationFunction.FUNCTION_NAME); - } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index 087bc55c54..81351569ce 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -6,16 +6,14 @@ package au.csiro.pathling.sql.dates; -import static au.csiro.pathling.utilities.Preconditions.checkNotNull; +import static au.csiro.pathling.utilities.Preconditions.checkUserInput; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.udf.SqlFunction2; import com.google.common.collect.ImmutableMap; import java.math.RoundingMode; -import java.time.temporal.ChronoUnit; -import java.time.temporal.Temporal; -import java.time.temporal.TemporalUnit; +import java.util.Calendar; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; @@ -24,65 +22,64 @@ import org.apache.spark.sql.Row; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.Quantity; -public abstract class TemporalArithmeticFunction implements +public abstract class TemporalArithmeticFunction implements SqlFunction2 { private static final long serialVersionUID = -5016153440496309996L; - static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() - .put("year", ChronoUnit.YEARS) - .put("years", ChronoUnit.YEARS) - .put("month", ChronoUnit.MONTHS) - .put("months", ChronoUnit.MONTHS) - .put("week", ChronoUnit.WEEKS) - .put("weeks", ChronoUnit.WEEKS) - .put("day", ChronoUnit.DAYS) - .put("days", ChronoUnit.DAYS) - .put("hour", ChronoUnit.HOURS) - .put("hours", ChronoUnit.HOURS) - .put("minute", ChronoUnit.MINUTES) - .put("minutes", ChronoUnit.MINUTES) - .put("second", ChronoUnit.SECONDS) - .put("seconds", ChronoUnit.SECONDS) - .put("millisecond", ChronoUnit.MILLIS) - .put("milliseconds", ChronoUnit.MILLIS) + static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("year", Calendar.YEAR) + .put("years", Calendar.YEAR) + .put("month", Calendar.MONTH) + .put("months", Calendar.MONTH) + .put("day", Calendar.DATE) + .put("days", Calendar.DATE) + .put("hour", Calendar.HOUR) + .put("hours", Calendar.HOUR) + .put("minute", Calendar.MINUTE) + .put("minutes", Calendar.MINUTE) + .put("second", Calendar.SECOND) + .put("seconds", Calendar.SECOND) + .put("millisecond", Calendar.MILLISECOND) + .put("milliseconds", Calendar.MILLISECOND) .build(); @Nonnull protected IntermediateType performAddition(@Nonnull final IntermediateType temporal, @Nonnull final Quantity calendarDuration) { - if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { - throw new IllegalArgumentException("Calendar duration must have a system of " - + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); - } - final long amountToAdd = calendarDuration.getValue() - .setScale(0, RoundingMode.HALF_UP) - .longValue(); - final TemporalUnit temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( - calendarDuration.getCode()); - checkNotNull(temporalUnit); - - //noinspection unchecked - return (IntermediateType) temporal.plus(amountToAdd, temporalUnit); + return performArithmetic(temporal, calendarDuration, false); } @Nonnull protected IntermediateType performSubtraction(@Nonnull final IntermediateType temporal, @Nonnull final Quantity calendarDuration) { + return performArithmetic(temporal, calendarDuration, true); + } + + @Nonnull + private IntermediateType performArithmetic(final @Nonnull IntermediateType temporal, + final @Nonnull Quantity calendarDuration, final boolean subtract) { if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { throw new IllegalArgumentException("Calendar duration must have a system of " + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); } - final long amountToAdd = calendarDuration.getValue() + final int amountToAdd = calendarDuration.getValue() .setScale(0, RoundingMode.HALF_UP) - .longValue(); - final TemporalUnit temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( + .intValue(); + final Integer temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( calendarDuration.getCode()); - - //noinspection unchecked - return (IntermediateType) temporal.minus(amountToAdd, temporalUnit); + checkUserInput(temporalUnit != null, + "Unsupported calendar duration unit: " + calendarDuration.getCode()); + + @SuppressWarnings("unchecked") + final IntermediateType result = (IntermediateType) temporal.copy(); + result.add(temporalUnit, subtract + ? amountToAdd * -1 + : amountToAdd); + return result; } protected abstract Function parseEncodedValue(); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java index 4130556f2f..8b59519127 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java @@ -6,8 +6,8 @@ package au.csiro.pathling.sql.dates.date; -import java.time.LocalDate; import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.DateType; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @@ -21,7 +21,7 @@ public class DateAddDurationFunction extends DateArithmeticFunction { public static final String FUNCTION_NAME = "date_add_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performAddition; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java index b9d60993b8..f5dbbcedd2 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java @@ -7,27 +7,21 @@ package au.csiro.pathling.sql.dates.date; import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.util.Date; import java.util.function.Function; import org.hl7.fhir.r4.model.DateType; -public abstract class DateArithmeticFunction extends TemporalArithmeticFunction { +public abstract class DateArithmeticFunction extends TemporalArithmeticFunction { private static final long serialVersionUID = 6759548804191034570L; @Override - protected Function parseEncodedValue() { - return LocalDate::parse; + protected Function parseEncodedValue() { + return DateType::new; } @Override - protected Function encodeResult() { - return (resultDate) -> new DateType( - new Date(resultDate.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) * 1000)) - .getValueAsString(); + protected Function encodeResult() { + return DateType::getValueAsString; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java index 5e1e96064f..5eac5038b5 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java @@ -6,8 +6,8 @@ package au.csiro.pathling.sql.dates.date; -import java.time.LocalDate; import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.DateType; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @@ -21,7 +21,7 @@ public class DateSubtractDurationFunction extends DateArithmeticFunction { public static final String FUNCTION_NAME = "date_subtract_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performSubtraction; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java index 024a15ac10..2b28b401f4 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java @@ -6,8 +6,8 @@ package au.csiro.pathling.sql.dates.datetime; -import java.time.temporal.Temporal; import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @@ -21,7 +21,7 @@ public class DateTimeAddDurationFunction extends DateTimeArithmeticFunction { public static final String FUNCTION_NAME = "datetime_add_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performAddition; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java index 81d59f0ff7..cdeb9b2ffb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java @@ -6,75 +6,23 @@ package au.csiro.pathling.sql.dates.datetime; -import static au.csiro.pathling.utilities.Preconditions.checkNotNull; - import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; -import ca.uhn.fhir.model.api.TemporalPrecisionEnum; -import com.google.common.collect.ImmutableMap; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.time.temporal.Temporal; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; import java.util.function.Function; -import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.DateTimeType; public abstract class DateTimeArithmeticFunction extends - TemporalArithmeticFunction { + TemporalArithmeticFunction { private static final long serialVersionUID = -6669722492626320119L; - static final Map JAVA_TO_HAPI_TIME_UNIT = new ImmutableMap.Builder() - .put(ChronoUnit.MINUTES, TemporalPrecisionEnum.MINUTE) - .put(ChronoUnit.SECONDS, TemporalPrecisionEnum.SECOND) - .put(ChronoUnit.MILLIS, TemporalPrecisionEnum.MILLI) - .build(); - @Override - protected Function parseEncodedValue() { - return (encoded) -> { - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern( - "yyyy-MM-dd'T'HH[:mm[:ss[.SSS][XXX]]"); - return (Temporal) formatter.parseBest(encoded, ZonedDateTime::from, LocalDateTime::from); - }; + protected Function parseEncodedValue() { + return DateTimeType::new; } @Override - protected Function encodeResult() { - return (temporal) -> { - final Instant instant = temporal instanceof ZonedDateTime - ? ((ZonedDateTime) temporal).toInstant() - : ((LocalDateTime) temporal).toInstant(ZoneOffset.UTC); - final BaseDateTimeType dateTime = new DateTimeType(Date.from(instant)); - - // Attempt to get the time zone from the parsed object. - try { - final TimeZone timeZone = TimeZone.getTimeZone(temporal.query(ZoneId::from)); - dateTime.setTimeZone(timeZone); - } catch (final Exception ignored) { - } - - // Find the appropriate precision for the result. - for (final ChronoUnit unit : List.of(ChronoUnit.MILLIS, ChronoUnit.SECONDS, - ChronoUnit.MINUTES)) { - if (temporal.isSupported(unit)) { - final TemporalPrecisionEnum precision = JAVA_TO_HAPI_TIME_UNIT.get(unit); - checkNotNull(precision); - dateTime.setPrecision(precision); - break; - } - } - - return dateTime.getValueAsString(); - }; + protected Function encodeResult() { + return DateTimeType::getValueAsString; } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java index 6f7be61ab7..57e3883292 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java @@ -6,8 +6,8 @@ package au.csiro.pathling.sql.dates.datetime; -import java.time.temporal.Temporal; import java.util.function.BiFunction; +import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.Quantity; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @@ -21,7 +21,7 @@ public class DateTimeSubtractDurationFunction extends DateTimeArithmeticFunction public static final String FUNCTION_NAME = "datetime_subtract_duration"; @Override - protected BiFunction getOperationFunction() { + protected BiFunction getOperationFunction() { return this::performSubtraction; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java deleted file mode 100644 index 86d8f7c898..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeAddDurationFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.sql.dates.time; - -import java.time.LocalTime; -import java.util.function.BiFunction; -import org.hl7.fhir.r4.model.Quantity; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -@Component -@Profile("core") -public class TimeAddDurationFunction extends TimeArithmeticFunction { - - private static final long serialVersionUID = 5839806160423512490L; - - public static final String FUNCTION_NAME = "time_add_duration"; - - @Override - protected BiFunction getOperationFunction() { - return this::performAddition; - } - - @Override - public String getName() { - return FUNCTION_NAME; - } - -} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java deleted file mode 100644 index 16aba024bc..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeArithmeticFunction.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.sql.dates.time; - -import au.csiro.pathling.sql.dates.TemporalArithmeticFunction; -import java.time.LocalTime; -import java.util.function.Function; -import org.hl7.fhir.r4.model.TimeType; - -public abstract class TimeArithmeticFunction extends TemporalArithmeticFunction { - - private static final long serialVersionUID = 7504107794184508523L; - - @Override - protected Function parseEncodedValue() { - return TimeFunction::parseEncodedTime; - } - - @Override - protected Function encodeResult() { - return (resultTime) -> new TimeType(resultTime.toString()) - .getValueAsString(); - } - -} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java deleted file mode 100644 index 2dc5fc5e1c..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeSubtractDurationFunction.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.sql.dates.time; - -import java.time.LocalTime; -import java.util.function.BiFunction; -import org.hl7.fhir.r4.model.Quantity; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -@Component -@Profile("core") -public class TimeSubtractDurationFunction extends TimeArithmeticFunction { - - private static final long serialVersionUID = 1909732257090337898L; - - public static final String FUNCTION_NAME = "time_subtract_duration"; - - @Override - protected BiFunction getOperationFunction() { - return this::performSubtraction; - } - - @Override - public String getName() { - return FUNCTION_NAME; - } - -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java index d390558f76..8323852618 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/DateArithmeticTest.java @@ -89,7 +89,7 @@ Stream parameters() throws ParseException { .withIdColumn(ID_ALIAS) .withColumn(DataTypes.StringType) .withRow("patient-1", "2015-02-07T13:28:17.000-05:00") - .withRow("patient-2", "2017-01-01T00:00:00+00:00") + .withRow("patient-2", "2017-01-01T00:00:00Z") .withRow("patient-3", "2025-06-21T00:15:00+10:00") .build(); final ElementPath dateTimePath = new ElementPathBuilder(spark) @@ -140,14 +140,10 @@ Stream parameters() throws ParseException { parameters.addAll(dateTimeSubtraction(dateTimePath, context)); parameters.addAll(dateAddition(datePath, context)); parameters.addAll(dateSubtraction(datePath, context)); - parameters.addAll(timeAddition(timePath, context)); - parameters.addAll(timeSubtraction(timePath, context)); parameters.addAll(dateTimeLiteralAddition(dateTimeLiteral, context)); parameters.addAll(dateTimeLiteralSubtraction(dateTimeLiteral, context)); parameters.addAll(dateLiteralAddition(dateLiteral, context)); parameters.addAll(dateLiteralSubtraction(dateLiteral, context)); - parameters.addAll(timeLiteralAddition(timeLiteral, context)); - parameters.addAll(timeLiteralSubtraction(timeLiteral, context)); return parameters.stream(); } @@ -159,9 +155,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000+00:00") - .withRow("patient-2", "2027-01-01T00:00:00.000+00:00") - .withRow("patient-3", "2035-06-20T14:15:00.000+00:00") + .withRow("patient-1", "2025-02-07T13:28:17.000-05:00") + .withRow("patient-2", "2027-01-01T00:00:00Z") + .withRow("patient-3", "2035-06-21T00:15:00+10:00") .build()) ); @@ -169,19 +165,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000+00:00") - .withRow("patient-2", "2017-10-01T00:00:00.000+00:00") - .withRow("patient-3", "2026-03-20T14:15:00.000+00:00") - .build()) - ); - - parameters.add(new TestParameters("DateTime + 2 weeks", dateTimePath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000+00:00") - .withRow("patient-2", "2017-01-15T00:00:00.000+00:00") - .withRow("patient-3", "2025-07-04T14:15:00.000+00:00") + .withRow("patient-1", "2015-11-07T13:28:17.000-05:00") + .withRow("patient-2", "2017-10-01T00:00:00Z") + .withRow("patient-3", "2026-03-21T00:15:00+10:00") .build()) ); @@ -189,9 +175,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000+00:00") - .withRow("patient-2", "2017-01-31T00:00:00.000+00:00") - .withRow("patient-3", "2025-07-20T14:15:00.000+00:00") + .withRow("patient-1", "2015-03-09T13:28:17.000-05:00") + .withRow("patient-2", "2017-01-31T00:00:00Z") + .withRow("patient-3", "2025-07-21T00:15:00+10:00") .build()) ); @@ -199,9 +185,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000+00:00") - .withRow("patient-2", "2017-01-01T12:00:00.000+00:00") - .withRow("patient-3", "2025-06-21T02:15:00.000+00:00") + .withRow("patient-1", "2015-02-08T01:28:17.000-05:00") + .withRow("patient-2", "2017-01-01T12:00:00Z") + .withRow("patient-3", "2025-06-21T12:15:00+10:00") .build()) ); @@ -209,9 +195,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000+00:00") - .withRow("patient-2", "2017-01-01T00:30:00.000+00:00") - .withRow("patient-3", "2025-06-20T14:45:00.000+00:00") + .withRow("patient-1", "2015-02-07T13:58:17.000-05:00") + .withRow("patient-2", "2017-01-01T00:30:00Z") + .withRow("patient-3", "2025-06-21T00:45:00+10:00") .build()) ); @@ -219,9 +205,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000+00:00") - .withRow("patient-2", "2017-01-01T00:00:10.000+00:00") - .withRow("patient-3", "2025-06-20T14:15:10.000+00:00") + .withRow("patient-1", "2015-02-07T13:28:27.000-05:00") + .withRow("patient-2", "2017-01-01T00:00:10Z") + .withRow("patient-3", "2025-06-21T00:15:10+10:00") .build()) ); @@ -229,9 +215,9 @@ Collection dateTimeAddition( QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300+00:00") - .withRow("patient-2", "2017-01-01T00:00:00.300+00:00") - .withRow("patient-3", "2025-06-20T14:15:00.300+00:00") + .withRow("patient-1", "2015-02-07T13:28:17.300-05:00") + .withRow("patient-2", "2017-01-01T00:00:00Z") + .withRow("patient-3", "2025-06-21T00:15:00+10:00") .build()) ); return parameters; @@ -244,9 +230,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000+00:00") - .withRow("patient-2", "2007-01-01T00:00:00.000+00:00") - .withRow("patient-3", "2015-06-20T14:15:00.000+00:00") + .withRow("patient-1", "2005-02-07T13:28:17.000-05:00") + .withRow("patient-2", "2007-01-01T00:00:00Z") + .withRow("patient-3", "2015-06-21T00:15:00+10:00") .build()) ); @@ -254,19 +240,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000+00:00") - .withRow("patient-2", "2016-04-01T00:00:00.000+00:00") - .withRow("patient-3", "2024-09-20T14:15:00.000+00:00") - .build()) - ); - - parameters.add(new TestParameters("DateTime - 2 weeks", dateTimePath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000+00:00") - .withRow("patient-2", "2016-12-18T00:00:00.000+00:00") - .withRow("patient-3", "2025-06-06T14:15:00.000+00:00") + .withRow("patient-1", "2014-05-07T13:28:17.000-05:00") + .withRow("patient-2", "2016-04-01T00:00:00Z") + .withRow("patient-3", "2024-09-21T00:15:00+10:00") .build()) ); @@ -274,9 +250,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000+00:00") - .withRow("patient-2", "2016-12-02T00:00:00.000+00:00") - .withRow("patient-3", "2025-05-21T14:15:00.000+00:00") + .withRow("patient-1", "2015-01-08T13:28:17.000-05:00") + .withRow("patient-2", "2016-12-02T00:00:00Z") + .withRow("patient-3", "2025-05-22T00:15:00+10:00") .build()) ); @@ -284,9 +260,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000+00:00") - .withRow("patient-2", "2016-12-31T12:00:00.000+00:00") - .withRow("patient-3", "2025-06-20T02:15:00.000+00:00") + .withRow("patient-1", "2015-02-07T01:28:17.000-05:00") + .withRow("patient-2", "2016-12-31T12:00:00Z") + .withRow("patient-3", "2025-06-20T12:15:00+10:00") .build()) ); @@ -294,9 +270,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 minutes", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000+00:00") - .withRow("patient-2", "2016-12-31T23:30:00.000+00:00") - .withRow("patient-3", "2025-06-20T13:45:00.000+00:00") + .withRow("patient-1", "2015-02-07T12:58:17.000-05:00") + .withRow("patient-2", "2016-12-31T23:30:00Z") + .withRow("patient-3", "2025-06-20T23:45:00+10:00") .build()) ); @@ -304,9 +280,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 seconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000+00:00") - .withRow("patient-2", "2016-12-31T23:59:50.000+00:00") - .withRow("patient-3", "2025-06-20T14:14:50.000+00:00") + .withRow("patient-1", "2015-02-07T13:28:07.000-05:00") + .withRow("patient-2", "2016-12-31T23:59:50Z") + .withRow("patient-3", "2025-06-21T00:14:50+10:00") .build()) ); @@ -314,9 +290,9 @@ Collection dateTimeSubtraction( QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", dateTimePath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700+00:00") - .withRow("patient-2", "2016-12-31T23:59:59.700+00:00") - .withRow("patient-3", "2025-06-20T14:14:59.700+00:00") + .withRow("patient-1", "2015-02-07T13:28:16.700-05:00") + .withRow("patient-2", "2016-12-31T23:59:59Z") + .withRow("patient-3", "2025-06-21T00:14:59+10:00") .build()) ); return parameters; @@ -345,16 +321,6 @@ Collection dateAddition( .build()) ); - parameters.add(new TestParameters("Date + 2 weeks", datePath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21") - .withRow("patient-2", "2017-01-15") - .withRow("patient-3", "2025-07-05") - .build()) - ); - parameters.add(new TestParameters("Date + 30 days", datePath, QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("+"), @@ -391,16 +357,6 @@ Collection dateSubtraction( .build()) ); - parameters.add(new TestParameters("Date - 2 weeks", datePath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", datePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24") - .withRow("patient-2", "2016-12-18") - .withRow("patient-3", "2025-06-07") - .build()) - ); - parameters.add(new TestParameters("Date - 30 days", datePath, QuantityLiteralPath.fromCalendarDurationString("30 days", datePath), context, Operator.getInstance("-"), @@ -421,9 +377,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2025-02-07T18:28:17.000+00:00") - .withRow("patient-2", "2025-02-07T18:28:17.000+00:00") - .withRow("patient-3", "2025-02-07T18:28:17.000+00:00") + .withRow("patient-1", "2025-02-07T18:28:17+00:00") + .withRow("patient-2", "2025-02-07T18:28:17+00:00") + .withRow("patient-3", "2025-02-07T18:28:17+00:00") .build()) ); @@ -431,19 +387,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-11-07T18:28:17.000+00:00") - .withRow("patient-2", "2015-11-07T18:28:17.000+00:00") - .withRow("patient-3", "2015-11-07T18:28:17.000+00:00") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 + 2 weeks", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21T18:28:17.000+00:00") - .withRow("patient-2", "2015-02-21T18:28:17.000+00:00") - .withRow("patient-3", "2015-02-21T18:28:17.000+00:00") + .withRow("patient-1", "2015-11-07T18:28:17+00:00") + .withRow("patient-2", "2015-11-07T18:28:17+00:00") + .withRow("patient-3", "2015-11-07T18:28:17+00:00") .build()) ); @@ -451,9 +397,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-03-09T18:28:17.000+00:00") - .withRow("patient-2", "2015-03-09T18:28:17.000+00:00") - .withRow("patient-3", "2015-03-09T18:28:17.000+00:00") + .withRow("patient-1", "2015-03-09T18:28:17+00:00") + .withRow("patient-2", "2015-03-09T18:28:17+00:00") + .withRow("patient-3", "2015-03-09T18:28:17+00:00") .build()) ); @@ -461,9 +407,9 @@ Collection dateTimeLiteralAddition( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-08T06:28:17.000+00:00") - .withRow("patient-2", "2015-02-08T06:28:17.000+00:00") - .withRow("patient-3", "2015-02-08T06:28:17.000+00:00") + .withRow("patient-1", "2015-02-08T06:28:17+00:00") + .withRow("patient-2", "2015-02-08T06:28:17+00:00") + .withRow("patient-3", "2015-02-08T06:28:17+00:00") .build()) ); @@ -473,9 +419,9 @@ Collection dateTimeLiteralAddition( context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:58:17.000+00:00") - .withRow("patient-2", "2015-02-07T18:58:17.000+00:00") - .withRow("patient-3", "2015-02-07T18:58:17.000+00:00") + .withRow("patient-1", "2015-02-07T18:58:17+00:00") + .withRow("patient-2", "2015-02-07T18:58:17+00:00") + .withRow("patient-3", "2015-02-07T18:58:17+00:00") .build()) ); @@ -485,9 +431,9 @@ Collection dateTimeLiteralAddition( context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:27.000+00:00") - .withRow("patient-2", "2015-02-07T18:28:27.000+00:00") - .withRow("patient-3", "2015-02-07T18:28:27.000+00:00") + .withRow("patient-1", "2015-02-07T18:28:27+00:00") + .withRow("patient-2", "2015-02-07T18:28:27+00:00") + .withRow("patient-3", "2015-02-07T18:28:27+00:00") .build()) ); @@ -497,9 +443,9 @@ Collection dateTimeLiteralAddition( context, Operator.getInstance("+"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:17.300+00:00") - .withRow("patient-2", "2015-02-07T18:28:17.300+00:00") - .withRow("patient-3", "2015-02-07T18:28:17.300+00:00") + .withRow("patient-1", "2015-02-07T18:28:17+00:00") + .withRow("patient-2", "2015-02-07T18:28:17+00:00") + .withRow("patient-3", "2015-02-07T18:28:17+00:00") .build()) ); @@ -513,9 +459,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("10 years", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2005-02-07T18:28:17.000+00:00") - .withRow("patient-2", "2005-02-07T18:28:17.000+00:00") - .withRow("patient-3", "2005-02-07T18:28:17.000+00:00") + .withRow("patient-1", "2005-02-07T18:28:17+00:00") + .withRow("patient-2", "2005-02-07T18:28:17+00:00") + .withRow("patient-3", "2005-02-07T18:28:17+00:00") .build()) ); @@ -523,19 +469,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("9 months", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2014-05-07T18:28:17.000+00:00") - .withRow("patient-2", "2014-05-07T18:28:17.000+00:00") - .withRow("patient-3", "2014-05-07T18:28:17.000+00:00") - .build()) - ); - - parameters.add(new TestParameters("@2015-02-07T18:28:17+00:00 - 2 weeks", dateTimeLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateTimeLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24T18:28:17.000+00:00") - .withRow("patient-2", "2015-01-24T18:28:17.000+00:00") - .withRow("patient-3", "2015-01-24T18:28:17.000+00:00") + .withRow("patient-1", "2014-05-07T18:28:17+00:00") + .withRow("patient-2", "2014-05-07T18:28:17+00:00") + .withRow("patient-3", "2014-05-07T18:28:17+00:00") .build()) ); @@ -543,9 +479,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("30 days", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-08T18:28:17.000+00:00") - .withRow("patient-2", "2015-01-08T18:28:17.000+00:00") - .withRow("patient-3", "2015-01-08T18:28:17.000+00:00") + .withRow("patient-1", "2015-01-08T18:28:17+00:00") + .withRow("patient-2", "2015-01-08T18:28:17+00:00") + .withRow("patient-3", "2015-01-08T18:28:17+00:00") .build()) ); @@ -553,9 +489,9 @@ Collection dateTimeLiteralSubtraction( QuantityLiteralPath.fromCalendarDurationString("12 hours", dateTimeLiteralPath), context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T06:28:17.000+00:00") - .withRow("patient-2", "2015-02-07T06:28:17.000+00:00") - .withRow("patient-3", "2015-02-07T06:28:17.000+00:00") + .withRow("patient-1", "2015-02-07T06:28:17+00:00") + .withRow("patient-2", "2015-02-07T06:28:17+00:00") + .withRow("patient-3", "2015-02-07T06:28:17+00:00") .build()) ); @@ -565,9 +501,9 @@ Collection dateTimeLiteralSubtraction( context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T17:58:17.000+00:00") - .withRow("patient-2", "2015-02-07T17:58:17.000+00:00") - .withRow("patient-3", "2015-02-07T17:58:17.000+00:00") + .withRow("patient-1", "2015-02-07T17:58:17+00:00") + .withRow("patient-2", "2015-02-07T17:58:17+00:00") + .withRow("patient-3", "2015-02-07T17:58:17+00:00") .build()) ); @@ -577,9 +513,9 @@ Collection dateTimeLiteralSubtraction( context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:07.000+00:00") - .withRow("patient-2", "2015-02-07T18:28:07.000+00:00") - .withRow("patient-3", "2015-02-07T18:28:07.000+00:00") + .withRow("patient-1", "2015-02-07T18:28:07+00:00") + .withRow("patient-2", "2015-02-07T18:28:07+00:00") + .withRow("patient-3", "2015-02-07T18:28:07+00:00") .build()) ); @@ -589,9 +525,9 @@ Collection dateTimeLiteralSubtraction( context, Operator.getInstance("-"), new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-07T18:28:16.700+00:00") - .withRow("patient-2", "2015-02-07T18:28:16.700+00:00") - .withRow("patient-3", "2015-02-07T18:28:16.700+00:00") + .withRow("patient-1", "2015-02-07T18:28:16+00:00") + .withRow("patient-2", "2015-02-07T18:28:16+00:00") + .withRow("patient-3", "2015-02-07T18:28:16+00:00") .build()) ); @@ -621,16 +557,6 @@ Collection dateLiteralAddition( .build()) ); - parameters.add(new TestParameters("@2015-02-07 + 2 weeks", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-02-21") - .withRow("patient-2", "2015-02-21") - .withRow("patient-3", "2015-02-21") - .build()) - ); - parameters.add(new TestParameters("@2015-02-07 + 30 days", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("+"), @@ -667,16 +593,6 @@ Collection dateLiteralSubtraction( .build()) ); - parameters.add(new TestParameters("@2015-02-07 - 2 weeks", dateLiteralPath, - QuantityLiteralPath.fromCalendarDurationString("2 weeks", dateLiteralPath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "2015-01-24") - .withRow("patient-2", "2015-01-24") - .withRow("patient-3", "2015-01-24") - .build()) - ); - parameters.add(new TestParameters("@2015-02-07 - 30 days", dateLiteralPath, QuantityLiteralPath.fromCalendarDurationString("30 days", dateLiteralPath), context, Operator.getInstance("-"), @@ -690,170 +606,6 @@ Collection dateLiteralSubtraction( return parameters; } - Collection timeAddition(final FhirPath timePath, - final ParserContext context) { - final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("Time + 6 hours", timePath, - QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "19:28:17") - .withRow("patient-2", "14:00") - .withRow("patient-3", "06:00") - .build())); - - parameters.add(new TestParameters("Time + 45 minutes", timePath, - QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "14:13:17") - .withRow("patient-2", "08:45") - .withRow("patient-3", "00:45") - .build())); - - parameters.add(new TestParameters("Time + 15 seconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "13:28:32") - .withRow("patient-2", "08:00:15") - .withRow("patient-3", "00:00:15") - .build())); - - parameters.add(new TestParameters("Time + 300 milliseconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "13:28:17.300") - .withRow("patient-2", "08:00:00.300") - .withRow("patient-3", "00:00:00.300") - .build())); - return parameters; - } - - Collection timeSubtraction(final FhirPath timePath, - final ParserContext context) { - final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("Time - 6 hours", timePath, - QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "07:28:17") - .withRow("patient-2", "02:00") - .withRow("patient-3", "18:00") - .build())); - - parameters.add(new TestParameters("Time - 45 minutes", timePath, - QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "12:43:17") - .withRow("patient-2", "07:15") - .withRow("patient-3", "23:15") - .build())); - - parameters.add(new TestParameters("Time - 15 seconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "13:28:02") - .withRow("patient-2", "07:59:45") - .withRow("patient-3", "23:59:45") - .build())); - - parameters.add(new TestParameters("Time - 300 milliseconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("300 milliseconds", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "13:28:16.700") - .withRow("patient-2", "07:59:59.700") - .withRow("patient-3", "23:59:59.700") - .build())); - return parameters; - } - - Collection timeLiteralAddition(final FhirPath timePath, - final ParserContext context) { - final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@T08:00 + 6 hours", timePath, - QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "14:00") - .withRow("patient-2", "14:00") - .withRow("patient-3", "14:00") - .build())); - - parameters.add(new TestParameters("@T08:00 + 45 minutes", timePath, - QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "08:45") - .withRow("patient-2", "08:45") - .withRow("patient-3", "08:45") - .build())); - - parameters.add(new TestParameters("@T08:00 + 15 seconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "08:00:15") - .withRow("patient-2", "08:00:15") - .withRow("patient-3", "08:00:15") - .build())); - - parameters.add(new TestParameters("@T08:00 + 70 milliseconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("70 milliseconds", timePath), context, - Operator.getInstance("+"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "08:00:00.070") - .withRow("patient-2", "08:00:00.070") - .withRow("patient-3", "08:00:00.070") - .build())); - return parameters; - } - - Collection timeLiteralSubtraction(final FhirPath timePath, - final ParserContext context) { - final List parameters = new ArrayList<>(); - parameters.add(new TestParameters("@T08:00 - 6 hours", timePath, - QuantityLiteralPath.fromCalendarDurationString("6 hours", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "02:00") - .withRow("patient-2", "02:00") - .withRow("patient-3", "02:00") - .build())); - - parameters.add(new TestParameters("@T08:00 - 45 minutes", timePath, - QuantityLiteralPath.fromCalendarDurationString("45 minutes", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "07:15") - .withRow("patient-2", "07:15") - .withRow("patient-3", "07:15") - .build())); - - parameters.add(new TestParameters("@T08:00 - 15 seconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("15 seconds", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "07:59:45") - .withRow("patient-2", "07:59:45") - .withRow("patient-3", "07:59:45") - .build())); - - parameters.add(new TestParameters("@T08:00 - 70 milliseconds", timePath, - QuantityLiteralPath.fromCalendarDurationString("70 milliseconds", timePath), context, - Operator.getInstance("-"), - new DatasetBuilder(spark).withIdColumn(ID_ALIAS).withColumn(DataTypes.StringType) - .withRow("patient-1", "07:59:59.930") - .withRow("patient-2", "07:59:59.930") - .withRow("patient-3", "07:59:59.930") - .build())); - return parameters; - } - @ParameterizedTest @MethodSource("parameters") void test(@Nonnull final TestParameters parameters) { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 7b6996f0d8..20030a52ed 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -21,13 +21,11 @@ import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; -import au.csiro.pathling.sql.dates.time.TimeAddDurationFunction; import au.csiro.pathling.sql.dates.time.TimeEqualsFunction; import au.csiro.pathling.sql.dates.time.TimeGreaterThanFunction; import au.csiro.pathling.sql.dates.time.TimeGreaterThanOrEqualToFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanOrEqualToFunction; -import au.csiro.pathling.sql.dates.time.TimeSubtractDurationFunction; import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; import au.csiro.pathling.terminology.CodingToLiteral; @@ -68,7 +66,6 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, final List sqlFunction2 = List.of(new DateTimeAddDurationFunction(), new DateTimeSubtractDurationFunction(), new DateAddDurationFunction(), new DateSubtractDurationFunction(), - new TimeAddDurationFunction(), new TimeSubtractDurationFunction(), new DateTimeEqualsFunction(), new DateTimeGreaterThanFunction(), new DateTimeGreaterThanOrEqualToFunction(), new DateTimeLessThanFunction(), new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index 89cb6b67a4..73b85e7318 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -15,8 +15,8 @@ literal expressions: - [String](#string) - [Integer](#integer) - [Decimal](#decimal) -- [Date](#date) - [DateTime](#datetime) +- [Date](#date) - [Time](#time) - [Quantity](#quantity) - [Coding](#coding) @@ -88,6 +88,27 @@ Examples: a scale of 6. +## DateTime + +The DateTime literal combines the [Date](#date) and [Time](#time) literals and +is a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). It uses the +`YYYY-MM-DDThh:mm:ss.ffff±hh:mm` format. `Z` is allowed as a synonym for the +zero (`+00:00`) UTC offset. + +Time zone is optional - if it is omitted, the system-configured time zone will +be assumed. Seconds and milliseconds precision are supported. Hours precision, +minutes precision and partial DateTime values (ending with `T`) are not +supported. + +Example: + +``` +@2014-01-25T14:30:14 // Seconds precision +@2014-01-25T14:30:14+10:00 // Seconds precision with UTC+10 timezone offset +@2014-01-25T14:30:14.559 // Milliseconds precision +@2014-01-25T14:30:14.559Z // Milliseconds precision with UTC timezone offset +``` + ## Date The Date type represents date and partial date values, without a time component. @@ -96,6 +117,10 @@ The Date literal is a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). It uses the `YYYY-MM-DD` format, though month and day parts are optional. +Some operations implicitly convert Date values to DateTime values, such as +comparison and arithmetic. Note that the Date will be assumed to be in the +system-configured time zone in these instances. + Examples: ``` @@ -112,34 +137,16 @@ The Time literal uses a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601): - A time begins with a `@T` -- It uses the `Thh:mm:ss` format, though minute and second are optional -- Milliseconds are not supported +- It uses the `Thh:mm:ss.fff` format, minutes, seconds and milliseconds are + optional Examples: ``` -@T07:30:14 // Seconds precision -@T14:30:14 // Minutes precision -@T14 // Hours precision -``` - -## DateTime - -The DateTime literal combines the [Date](#date) and [Time](#time) literals and -is a subset of [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). It uses the -`YYYY-MM-DDThh:mm:ss.ffff±hh:mm` format. `Z` is allowed as a synonym for the -zero (`+00:00`) UTC offset. - -Time zone is optional. Seconds and milliseconds precision are supported. Hours -precision and partial DateTime values (ending with `T`) are not supported. - -Example: - -``` -@2014-01-25T14:30:14 // Seconds precision -@2014-01-25T14:30:14+10:00 // Seconds precision with UTC+10 timezone offset -@2014-01-25T14:30:14.559 // Milliseconds precision -@2014-01-25T14:30:14.559Z // Milliseconds precision with UTC timezone offset +@T07:30:14.350 // Milliseconds precision +@T07:30:14 // Seconds precision +@T14:30:14 // Minutes precision +@T14 // Hours precision ``` ## Quantity diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index e1d5d8d527..c5e10abcd6 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -112,21 +112,21 @@ See also: [Math](https://hl7.org/fhirpath/#math) The following operators are supported for date arithmetic: -- `+` - Add a duration to a [Date](./data-types.html#date), - [DateTime](./data-types.html#datetime) or [Time](./data-types.html#time) -- `-` - Subtract a duration from a [Date](./data-types.html#date), - [DateTime](./data-types.html#datetime) or [Time](./data-types.html#time) +- `+` - Add a duration to a [Date](./data-types.html#date) or + [DateTime](./data-types.html#datetime) +- `-` - Subtract a duration from a [Date](./data-types.html#date) or + [DateTime](./data-types.html#datetime) -Date arithmetic always has a `DateTime`, `Date` or `Time` on the left-hand side, -and a duration on the right-hand side. The duration operand is a +Date arithmetic always has a `DateTime` or `Date` on the left-hand side, and a +duration on the right-hand side. The duration operand is a [calendar duration literal](./data-types.html#quantity). The use of UCUM units is not supported with these operators. The `Date` or `DateTime` operand must be singular. If it is an empty collection, the operator will return an empty collection. -The `Date` type does not support arithmetic with durations below `day/days`. The -`Time` type does not support arithmetic with durations above `hour/hours`. +The use of arithmetic with the [Time](./data-types.html#time) type is not +supported. See also: [Date/Time Arithmetic](https://hl7.org/fhirpath/#datetime-arithmetic) From fcd9ede888518d87d68a7e32a50a8ebbd5c2370a Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 2 May 2022 16:13:58 +1000 Subject: [PATCH 21/83] Fix GroupingLiteralTest --- .../GroupingLiteralTest/DateTimeLiteral.Parameters.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json b/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json index d34c1cffd6..e3c4b55d57 100644 --- a/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json +++ b/fhir-server/src/test/resources/responses/GroupingLiteralTest/DateTimeLiteral.Parameters.json @@ -6,7 +6,7 @@ "part": [ { "name": "label", - "valueDateTime": "2015-02-08T18:28:17+00:00" + "valueDateTime": "2015-02-08T13:28:17-05:00" }, { "name": "result", @@ -14,7 +14,7 @@ }, { "name": "drillDown", - "valueString": "(@2015-02-08T13:28:17-05:00) = @2015-02-08T18:28:17+00:00" + "valueString": "(@2015-02-08T13:28:17-05:00) = @2015-02-08T13:28:17-05:00" } ] } From d3deb5205d44cd1d69a46d8e61bc277f46cec24f Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 2 May 2022 16:14:24 +1000 Subject: [PATCH 22/83] Add DateTimeArithmeticParserTest --- .../fhirpath/element/DateTimePath.java | 3 +- .../fhirpath/literal/LiteralPath.java | 3 +- .../parser/DateTimeArithmeticParserTest.java | 62 +++++ .../ParserTest/ageAtTimeOfEncounter.csv | 217 ++++++++++++++++++ .../ParserTest/lengthOfEncounter.csv | 217 ++++++++++++++++++ 5 files changed, 498 insertions(+), 4 deletions(-) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java create mode 100644 fhir-server/src/test/resources/responses/ParserTest/ageAtTimeOfEncounter.csv create mode 100644 fhir-server/src/test/resources/responses/ParserTest/lengthOfEncounter.csv diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index de42862b5c..b7c34d395e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -31,7 +31,6 @@ import java.util.Optional; import java.util.function.Function; import javax.annotation.Nonnull; -import javolution.testing.AssertionException; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -125,7 +124,7 @@ public static Function buildComparison(@Nonnull final Compar functionName = DateTimeGreaterThanOrEqualToFunction.FUNCTION_NAME; break; default: - throw new AssertionException("Unsupported operation: " + operation); + throw new AssertionError("Unsupported operation: " + operation); } return callUDF(functionName, source.getValueColumn(), target.getValueColumn()); }; diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java index b750dcc9ed..0aab3bd0ca 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java @@ -19,7 +19,6 @@ import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; -import javolution.testing.AssertionException; import lombok.Getter; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; @@ -132,7 +131,7 @@ public static String expressionFor(@Nonnull final Dataset dataset, .filter(c -> c.getParameterTypes()[1] == Column.class) .filter(c -> Type.class.isAssignableFrom(c.getParameterTypes()[2])) .findFirst() - .orElseThrow(() -> new AssertionException( + .orElseThrow(() -> new AssertionError( "No suitable constructor found for " + literalPathClass)); final LiteralPath literalPath = constructor.newInstance(dataset, idColumn, literalValue); return literalPath.getExpression(); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java new file mode 100644 index 0000000000..3cae75323e --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.parser; + +import static au.csiro.pathling.test.helpers.TestHelpers.mockEmptyResource; + +import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.element.BooleanPath; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import java.util.Collections; +import org.hl7.fhir.r4.model.Enumerations.ResourceType; +import org.junit.jupiter.api.Test; + +public class DateTimeArithmeticParserTest extends ParserTest { + + @Test + void lengthOfEncounter() { + final ResourcePath subjectResource = ResourcePath + .build(fhirContext, database, ResourceType.ENCOUNTER, ResourceType.ENCOUNTER.toCode(), + true); + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .terminologyClientFactory(terminologyServiceFactory) + .database(database) + .inputContext(subjectResource) + .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) + .build(); + parser = new Parser(parserContext); + + mockEmptyResource(database, spark, fhirEncoders, ResourceType.GROUP); + + assertThatResultOf("(period.start + 20 minutes) > period.end") + .isElementPath(BooleanPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/lengthOfEncounter.csv"); + } + + @Test + void ageAtTimeOfEncounter() { + final ResourcePath subjectResource = ResourcePath + .build(fhirContext, database, ResourceType.ENCOUNTER, ResourceType.ENCOUNTER.toCode(), + true); + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .terminologyClientFactory(terminologyServiceFactory) + .database(database) + .inputContext(subjectResource) + .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) + .build(); + parser = new Parser(parserContext); + + mockEmptyResource(database, spark, fhirEncoders, ResourceType.GROUP); + + assertThatResultOf("period.start > (subject.resolve().ofType(Patient).birthDate + 60 years)") + .isElementPath(BooleanPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/ageAtTimeOfEncounter.csv"); + } + +} diff --git a/fhir-server/src/test/resources/responses/ParserTest/ageAtTimeOfEncounter.csv b/fhir-server/src/test/resources/responses/ParserTest/ageAtTimeOfEncounter.csv new file mode 100644 index 0000000000..c3c42afdb5 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/ageAtTimeOfEncounter.csv @@ -0,0 +1,217 @@ +0134901a-4a69-4b39-92ea-d6f48ec83c6e,false +01ac1476-0c9c-4cd7-82a6-4ff526018e9a,false +0250a217-a663-403b-a708-5d14eadf0c40,false +02b993c0-b358-481c-b0d0-0767387feb9f,false +0364073f-91d8-47a1-b8b0-107c6318a691,false +04945715-8ad5-4d8f-bfda-ae26662a3610,false +04d88eb4-fb1b-4fa1-802a-5624f4c61b32,false +07bb3bb3-09e6-4abf-85f4-8ad113e7afa5,false +0af4a77e-892e-4b3f-9110-4c5224782250,false +0c315d37-c758-4cf5-a7cb-cbf712fa7dfd,false +102e16c5-4920-4dad-b142-0b280b2aacad,false +11dafd5c-85e7-4275-a147-89a63641e35e,false +12b212d8-bc6e-415a-93b0-594381726668,false +12df1bed-5472-4c48-8e88-2cbb3e5eab33,false +151e3048-66ad-4411-8753-677877e3bf0a,false +16a3408d-c927-4d02-a1ae-cad32419caab,false +16d280a2-c61f-487f-9034-22e884158969,false +17bcf2ea-d921-4715-91c5-6b15226b33d3,false +18ebf5ea-b0a2-4333-9e9c-40217de809ff,false +19502b5a-2031-487d-9d9c-2001f959408b,false +1cb88e7a-3db6-4a88-9b68-b0e1492114bc,false +1db1c71b-9eeb-4a98-abc1-eb699b38510c,false +1f3443ee-d15e-4894-b18e-185cfadfcdea,false +1fd714a6-48b4-425a-99b3-ba5c1a995cce,false +202131ae-78bb-4104-89b0-fb90645760f6,false +20fbcb6e-d793-4b05-8e29-8ff82572b0db,false +21995497-0eb8-4c0b-8c23-e6a829f3d596,false +21c6e0fc-bf47-4f80-b1ff-85b940445bdf,false +224e538d-3622-4f5e-b772-211ed358d8de,false +23780032-f3bb-4b40-af2e-3fa6b1677376,false +24c4dd5b-748b-4165-a0cc-2867b76c2dc2,false +2514cdf0-1216-4c5e-bfa8-0c11c20c6b93,false +274a2e49-9170-4945-bca2-ab6ef4bded75,false +2830e99a-e6b0-411b-a051-7ea1e9f815d1,false +29d4e919-2bd2-44e6-bb8a-380a780f60ff,false +2aff9edd-def2-487a-b435-a162e11a303c,false +2bd766e5-60e4-4469-bb97-54f0503a1eb0,false +2bddd98c-86e3-4ae0-9dc6-87da16ab6267,false +2cd8b9e4-2881-45a4-b1a6-c3a38e9d59d5,false +2d2e07ec-e697-4384-a679-867e93740921,false +2de1f829-c9d2-4f22-bf39-536dcb82fc3e,false +2e8eb256-84c9-499a-8bf6-bb39504373e1,false +2ec33cf5-d22d-414b-bf5d-e4b4e9997c0f,false +2ee2ab26-09fb-4e12-bf21-9fed3b4c38de,false +2fc75f11-2097-4e1e-8390-dbff3e844b7a,false +2ffd6142-b75b-406f-bc06-cdb1c02eaefc,false +308eba53-29fa-489a-97dc-741b077841a7,false +30ae4a13-0cc6-4d12-a5dd-fa9288efcc09,false +31d4c712-75b1-4ecb-9d1a-ccc4b8453826,false +3397a796-894d-409c-97de-a9e6f3f88198,false +359cb842-c25b-429f-ba9b-d52f171e5631,false +36160225-76a2-4cf1-8bf8-5a3d3f2ec9d7,false +374fd699-6b74-4500-9d60-2473c7e0364f,false +395019b6-84e8-4919-8b8b-ac64e4a6eade,false +39efc9b8-a75f-43a3-98a2-5bdc1c8207d2,false +3ddf46fe-b5be-456d-810f-ea6a7402236d,false +4148ac36-1dcb-4e96-89cd-355e7e8f919b,false +41d26b1c-57b9-408c-b955-bf0ee1db4809,false +4215b5d0-bdd9-4222-a04a-81bb637d60af,false +42a9a333-d09d-4b9e-af96-1f02af26bac3,true +42aa48c7-68cc-4bee-9730-cff29f0e36c5,false +4506aa76-9d0d-40e4-85bc-5735f74522a0,false +45b25ced-6eed-4fca-8ec1-b7d32ea13efe,false +45ec3b9e-eb82-48b4-b4b6-bd305afdf8b3,false +48cc2275-a478-4ade-b076-cf38e1d16017,false +498f4b18-bd4c-470d-9cde-2fca70ec8964,false +4a464bb5-9373-4f1b-9c03-053c64797114,false +4adcaf72-5241-4877-b61c-f34060576c50,false +4cc605d0-d0c6-4de3-849a-f5f92b35d2a0,false +4cd95073-6db3-4eb2-ac4b-0aacb182b352,false +4cfe72fe-21a0-4201-87b6-ef93695d2495,false +4db144d3-a596-4718-a50a-8ac9175ad386,false +4e6b69c7-61f9-4ff5-8c33-ad400edb9b06,false +4f14ca7f-5332-4263-a7b2-f0a838380309,false +4f342a71-dec3-403e-970b-e4c21b9eb98b,false +4f3d1e93-a28f-446e-91af-2baf0168b82b,false +50eac25a-3761-4de3-8a69-aae52e658300,true +51241857-a7a4-4517-96e3-21f797581f89,false +522e2abe-ad96-4d1a-b5a5-748faa997531,false +54c750aa-570a-4e8e-9a02-418781923e39,false +55f17f44-287f-41aa-a1bc-d1c0104af36e,false +56999896-e36b-4e63-a6b6-dbb85dfe1936,false +578cc35c-0982-4d72-a729-b304c026f075,false +5abe533b-ae5f-47ad-8746-f60caf7483c0,false +5cedf18a-fc44-4c39-a80f-2106a0d76934,false +5e1fe0f2-4517-462b-8287-7824f9a60f4f,false +5f64131b-d410-449a-9de6-35415919fec5,false +62059efc-7607-41c9-a3ba-70948f6a8a1d,false +634f0889-3a83-484a-bcc8-86f48e333c7b,false +67e5cf95-236b-4f75-abc5-034ca2966448,false +67fc12fc-bb9f-40f1-9051-127a788b3081,false +683714ad-5194-49e7-9b8f-4e306cf68ae1,false +6954c5c2-dd77-490a-9152-f1d78eff913d,false +6a5d8a68-bbe0-4408-ada5-a843ee6bd9b0,false +6af73cae-2e4e-485f-a74d-6f4307eb3af3,false +6c84348c-5065-4e89-9412-bfda023683f2,false +6d47a2fe-3645-4c5c-bebe-1b822eff197f,false +6e187f47-91a1-4217-a185-31b6f01db225,false +720f7ed6-7b2e-41dc-a0e0-724a3332aa24,false +721e4027-a080-40d8-bc4a-43cd33477611,false +72788e7f-1bf1-40b5-a1f1-27c669dc7278,false +72c611bb-471e-40f2-a6a8-0ae7b7b6b63e,false +749babeb-6883-4bd4-92a5-bec52769071c,false +75a09d4f-eef2-4404-a386-dfcb408ec9ed,false +777aa5f2-2a09-4aed-92ea-912694e28b48,false +79be3b69-23d8-4408-8f01-68d993e63b6c,false +7ac56d5a-32a7-4b40-a956-356e3006faed,false +7e6fe8ca-a95e-4102-8b18-e6e858cd15a8,false +7e7322a6-7912-4101-b3be-d134245a0626,false +7ea43fc6-b1f7-46be-a308-5ecb275a1081,false +7f410064-638a-4477-85c4-97fc2bb14a49,false +8089a9ac-9e71-4487-a231-f720e4bc8d3e,false +836b2e59-fb97-4014-ab0c-d5a5dc750969,false +85481b3f-c75e-40cd-bacd-b0afb79e893c,false +85c997bf-63d9-4ebb-8e0b-320de3dddc6c,false +85e41a01-04e5-44d2-8b3a-4d3ad7b0f87b,false +86415df5-7e47-4637-9e09-aaad9e7628d1,false +8743e69b-17aa-44ff-b8fa-4f582665efc6,false +87b04a97-1d33-4548-aec9-3d2856070702,false +88b16960-bfff-41d0-81e4-9a4630834a00,false +89f260bb-68b4-4dec-91f4-d52e673e0ff7,false +8a1908f7-47cc-4f82-859c-781a193e9901,false +8bc7af32-e5ad-4849-ba4d-b446db833ab4,false +8be81657-8ba6-4ec5-8ff9-4809367096ea,false +8dc2ce26-aec7-43c0-9771-350af4257ad8,false +8e9e848c-0547-4f2c-8b5f-e33d79bbed67,false +8f76d729-9f74-4cfd-be2a-8b033b568e18,false +900ce9fe-02d3-476b-902a-9467767ecdcf,false +922e2443-9cc5-421f-8310-9cd23f6e9f2d,false +96b2282d-1384-4cfb-9958-f009fb501626,false +97b42d67-8691-433d-8a31-dada076162ae,false +984067fc-3dfc-47b7-8ee0-4d9b27d43860,false +990c21ab-433b-4920-873e-f9dfc7103d9d,false +99fc3558-79f3-4d14-9aa1-ff63f621ecd9,false +9b2eb632-6f1a-4e97-912b-3c8378a1b11c,false +9b4f52b0-51fa-4ea6-b02c-2ed8478066d3,false +9bc516d4-7c82-4340-a90c-bb493f96fbe4,false +9e0b8dc3-65c4-490d-97c0-d5e3db944de5,false +9ed8703e-3b40-424c-9e98-b3b404051f99,false +a1228f20-7c50-4d3a-b88c-2d277ca79d79,false +a221198d-000e-497b-8b70-bb4d37f7bbe1,false +a2c3ee1d-ec35-4b26-9bab-12de3f47d604,false +a6790cbd-8349-432e-a976-5dfc07451203,false +a6b084fb-ada7-480a-9adc-f180d4eb1e2a,false +a6cb423a-7571-46df-83e2-ccc6820adb36,false +a89c0c42-c19f-4d7f-a15e-d04e043c92f6,false +acb2329d-b1c3-45c3-ad28-91a09aa521e1,false +add6d754-a075-4693-a33a-c061c9a368ff,false +ae7ddaef-4911-418c-8401-d978617e145b,false +ae9e663e-8c15-43c9-8a88-52b61cbf07a9,false +b00df815-7528-4967-bfe2-311130d91c21,false +b07fb98d-2da4-423a-83b4-7a673de93096,false +b0d337c5-3555-4817-ba59-4ceea5ad8a91,false +b1ccd55a-6b88-4240-b20c-ca893f36e23b,false +b1d4dc41-07ae-44db-ae17-1df86c5a62cf,false +b5d06aa6-b5c6-4782-a0f8-dbf4121e1baa,false +b6900c21-d310-4fb4-8b8f-176a09c91989,false +b7772635-1801-48e4-a442-f4aaa6860544,false +b870d2cb-ea6a-4b67-8de5-ce1224a18bf2,false +bac374c0-50e8-4a5c-947b-82a8e9a747a9,false +baf48725-f0f9-4357-a71d-b1104910deae,false +bb1a7816-4798-4f58-9ce4-c5bd3af682a8,true +bbd0f936-0cb9-4cc9-858c-2926b2b1c1eb,false +bf895c48-1d52-4780-8e9a-532d69356d28,false +bfc93cb7-e53a-49f7-b86e-81d6a13cfdea,false +c06a9f9b-90fa-4bd6-9581-0c6950f03e4a,false +c3ec1f2e-1411-41e4-8370-acd5a8f68cf4,false +c57d51a7-45de-41b9-92fc-efd0634effaf,false +c5c6eed5-aec6-4b8a-b689-adbb219c2267,false +c892b1d7-7485-407d-8883-54fb7fecebc1,false +c90aae7e-5704-4e13-8794-41ab377193bd,false +ca3e0486-fd6b-4a13-a741-3a47760b412d,false +ca7f269d-3794-4994-8c46-d8e9f2aa6378,false +cc56af6d-25c6-480e-9767-da02a55551da,false +cdb8493c-e3b0-447f-b16b-e58dd64660f3,false +cf67a8d4-48e2-4dc9-a521-6aabcff0330b,false +cfcbe34a-edc5-4442-bc05-5927fa00336b,false +d05461d6-13bf-402c-890d-db6404933f7c,false +d080c116-681a-494a-a928-45924109b49d,false +d535b213-2abc-438f-aa6a-c06476d60b09,false +d6988116-3c63-44f8-9f94-9e9d51cc5a37,false +d79728e1-8834-4f4a-8edc-ce18aca18be6,true +d7a48a26-7731-4046-bf22-78f0b8084eb9,false +d90676d2-ce74-4867-a00f-6dbffa7c00be,false +d9e7dd88-2d7c-4e48-8cf9-9e33f729b7c2,false +db79f75d-af3c-4f82-b4e5-9b5d26598eef,false +def5fd23-2b74-4c64-85e9-fac27e626bb7,false +dfe38e2d-23d5-45c6-86bd-f83fe3b6a1d8,false +e0dfbc08-45ad-4a81-b648-7e65c066f673,false +e1040d58-53bc-4467-8101-ca53b772cede,false +e1c2ad9f-6f61-4f16-805a-2f405b63659a,false +e2387a81-5ff5-4daf-b2e8-881998d0d917,false +e26ae29a-213a-4f79-98c2-cc81cf2f451d,false +e363eb67-64c7-440f-93fd-5c8787c69a85,false +e4358f28-62a8-4e06-b2bd-cb00d3310349,false +e7f68d35-ae55-4fa4-b0be-0672a5a7186a,false +e8c41168-a093-40eb-8baa-6802d24f554a,false +e8eba267-fecb-477e-9625-771fbcfc3cba,false +ede7745a-8f3e-4e20-9f16-33756be7ab6c,false +ee24bf89-81a3-4259-a382-86917f3829d4,false +f1293c5e-c997-4ba9-8bfd-438e7a840dc8,false +f18b08bd-40da-43c1-87ec-4ba23542635f,false +f19eefca-c1ad-4860-bdda-4674a631d464,false +f28931e5-1f35-4f9f-a02e-78398735ea67,false +f29330d0-5b32-4f75-b558-a1c7f05c0b4a,false +f2c9ca81-8b1e-4873-bd8a-1a6e1411193c,false +f3aa491a-4dbd-4436-b674-18b2177dcf18,false +f3c2f842-6aa3-42c9-a0f3-895c40456868,false +f3eb0c38-2571-44e7-be39-732eb52cf1df,false +f6d3ff7b-f6e1-416a-bb05-73dfa8d129b5,false +f77a8561-5adc-452a-ac9a-76273b6a6678,true +f883f2ca-d523-4c9f-86b2-0add137901de,false +fc4d37ec-22b7-4ea0-a34e-dcd4beb2001c,false +fc771883-3bd6-46d9-9626-a130e1b902de,false +fcab9efe-fb05-42d6-b366-ce6db1cc4093,false +fcc916dd-5e4f-4bf7-9aef-c906a36d7301,false +fce20464-ff98-4051-8714-6b9584e52740,false diff --git a/fhir-server/src/test/resources/responses/ParserTest/lengthOfEncounter.csv b/fhir-server/src/test/resources/responses/ParserTest/lengthOfEncounter.csv new file mode 100644 index 0000000000..bbfdbb7687 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/lengthOfEncounter.csv @@ -0,0 +1,217 @@ +0134901a-4a69-4b39-92ea-d6f48ec83c6e,true +01ac1476-0c9c-4cd7-82a6-4ff526018e9a,true +0250a217-a663-403b-a708-5d14eadf0c40,true +02b993c0-b358-481c-b0d0-0767387feb9f,true +0364073f-91d8-47a1-b8b0-107c6318a691,true +04945715-8ad5-4d8f-bfda-ae26662a3610,true +04d88eb4-fb1b-4fa1-802a-5624f4c61b32,true +07bb3bb3-09e6-4abf-85f4-8ad113e7afa5,true +0af4a77e-892e-4b3f-9110-4c5224782250,true +0c315d37-c758-4cf5-a7cb-cbf712fa7dfd,false +102e16c5-4920-4dad-b142-0b280b2aacad,false +11dafd5c-85e7-4275-a147-89a63641e35e,true +12b212d8-bc6e-415a-93b0-594381726668,true +12df1bed-5472-4c48-8e88-2cbb3e5eab33,false +151e3048-66ad-4411-8753-677877e3bf0a,false +16a3408d-c927-4d02-a1ae-cad32419caab,false +16d280a2-c61f-487f-9034-22e884158969,false +17bcf2ea-d921-4715-91c5-6b15226b33d3,true +18ebf5ea-b0a2-4333-9e9c-40217de809ff,false +19502b5a-2031-487d-9d9c-2001f959408b,true +1cb88e7a-3db6-4a88-9b68-b0e1492114bc,false +1db1c71b-9eeb-4a98-abc1-eb699b38510c,false +1f3443ee-d15e-4894-b18e-185cfadfcdea,false +1fd714a6-48b4-425a-99b3-ba5c1a995cce,true +202131ae-78bb-4104-89b0-fb90645760f6,true +20fbcb6e-d793-4b05-8e29-8ff82572b0db,true +21995497-0eb8-4c0b-8c23-e6a829f3d596,true +21c6e0fc-bf47-4f80-b1ff-85b940445bdf,true +224e538d-3622-4f5e-b772-211ed358d8de,false +23780032-f3bb-4b40-af2e-3fa6b1677376,true +24c4dd5b-748b-4165-a0cc-2867b76c2dc2,false +2514cdf0-1216-4c5e-bfa8-0c11c20c6b93,false +274a2e49-9170-4945-bca2-ab6ef4bded75,true +2830e99a-e6b0-411b-a051-7ea1e9f815d1,true +29d4e919-2bd2-44e6-bb8a-380a780f60ff,true +2aff9edd-def2-487a-b435-a162e11a303c,false +2bd766e5-60e4-4469-bb97-54f0503a1eb0,false +2bddd98c-86e3-4ae0-9dc6-87da16ab6267,false +2cd8b9e4-2881-45a4-b1a6-c3a38e9d59d5,true +2d2e07ec-e697-4384-a679-867e93740921,false +2de1f829-c9d2-4f22-bf39-536dcb82fc3e,true +2e8eb256-84c9-499a-8bf6-bb39504373e1,true +2ec33cf5-d22d-414b-bf5d-e4b4e9997c0f,true +2ee2ab26-09fb-4e12-bf21-9fed3b4c38de,true +2fc75f11-2097-4e1e-8390-dbff3e844b7a,false +2ffd6142-b75b-406f-bc06-cdb1c02eaefc,false +308eba53-29fa-489a-97dc-741b077841a7,true +30ae4a13-0cc6-4d12-a5dd-fa9288efcc09,false +31d4c712-75b1-4ecb-9d1a-ccc4b8453826,false +3397a796-894d-409c-97de-a9e6f3f88198,true +359cb842-c25b-429f-ba9b-d52f171e5631,true +36160225-76a2-4cf1-8bf8-5a3d3f2ec9d7,true +374fd699-6b74-4500-9d60-2473c7e0364f,false +395019b6-84e8-4919-8b8b-ac64e4a6eade,false +39efc9b8-a75f-43a3-98a2-5bdc1c8207d2,true +3ddf46fe-b5be-456d-810f-ea6a7402236d,true +4148ac36-1dcb-4e96-89cd-355e7e8f919b,false +41d26b1c-57b9-408c-b955-bf0ee1db4809,true +4215b5d0-bdd9-4222-a04a-81bb637d60af,false +42a9a333-d09d-4b9e-af96-1f02af26bac3,true +42aa48c7-68cc-4bee-9730-cff29f0e36c5,false +4506aa76-9d0d-40e4-85bc-5735f74522a0,true +45b25ced-6eed-4fca-8ec1-b7d32ea13efe,false +45ec3b9e-eb82-48b4-b4b6-bd305afdf8b3,true +48cc2275-a478-4ade-b076-cf38e1d16017,true +498f4b18-bd4c-470d-9cde-2fca70ec8964,false +4a464bb5-9373-4f1b-9c03-053c64797114,true +4adcaf72-5241-4877-b61c-f34060576c50,false +4cc605d0-d0c6-4de3-849a-f5f92b35d2a0,false +4cd95073-6db3-4eb2-ac4b-0aacb182b352,false +4cfe72fe-21a0-4201-87b6-ef93695d2495,false +4db144d3-a596-4718-a50a-8ac9175ad386,false +4e6b69c7-61f9-4ff5-8c33-ad400edb9b06,false +4f14ca7f-5332-4263-a7b2-f0a838380309,false +4f342a71-dec3-403e-970b-e4c21b9eb98b,true +4f3d1e93-a28f-446e-91af-2baf0168b82b,true +50eac25a-3761-4de3-8a69-aae52e658300,true +51241857-a7a4-4517-96e3-21f797581f89,true +522e2abe-ad96-4d1a-b5a5-748faa997531,true +54c750aa-570a-4e8e-9a02-418781923e39,true +55f17f44-287f-41aa-a1bc-d1c0104af36e,true +56999896-e36b-4e63-a6b6-dbb85dfe1936,false +578cc35c-0982-4d72-a729-b304c026f075,true +5abe533b-ae5f-47ad-8746-f60caf7483c0,false +5cedf18a-fc44-4c39-a80f-2106a0d76934,true +5e1fe0f2-4517-462b-8287-7824f9a60f4f,true +5f64131b-d410-449a-9de6-35415919fec5,true +62059efc-7607-41c9-a3ba-70948f6a8a1d,true +634f0889-3a83-484a-bcc8-86f48e333c7b,true +67e5cf95-236b-4f75-abc5-034ca2966448,true +67fc12fc-bb9f-40f1-9051-127a788b3081,false +683714ad-5194-49e7-9b8f-4e306cf68ae1,true +6954c5c2-dd77-490a-9152-f1d78eff913d,true +6a5d8a68-bbe0-4408-ada5-a843ee6bd9b0,false +6af73cae-2e4e-485f-a74d-6f4307eb3af3,true +6c84348c-5065-4e89-9412-bfda023683f2,false +6d47a2fe-3645-4c5c-bebe-1b822eff197f,false +6e187f47-91a1-4217-a185-31b6f01db225,true +720f7ed6-7b2e-41dc-a0e0-724a3332aa24,true +721e4027-a080-40d8-bc4a-43cd33477611,true +72788e7f-1bf1-40b5-a1f1-27c669dc7278,true +72c611bb-471e-40f2-a6a8-0ae7b7b6b63e,true +749babeb-6883-4bd4-92a5-bec52769071c,true +75a09d4f-eef2-4404-a386-dfcb408ec9ed,true +777aa5f2-2a09-4aed-92ea-912694e28b48,true +79be3b69-23d8-4408-8f01-68d993e63b6c,true +7ac56d5a-32a7-4b40-a956-356e3006faed,false +7e6fe8ca-a95e-4102-8b18-e6e858cd15a8,false +7e7322a6-7912-4101-b3be-d134245a0626,false +7ea43fc6-b1f7-46be-a308-5ecb275a1081,false +7f410064-638a-4477-85c4-97fc2bb14a49,false +8089a9ac-9e71-4487-a231-f720e4bc8d3e,false +836b2e59-fb97-4014-ab0c-d5a5dc750969,true +85481b3f-c75e-40cd-bacd-b0afb79e893c,true +85c997bf-63d9-4ebb-8e0b-320de3dddc6c,true +85e41a01-04e5-44d2-8b3a-4d3ad7b0f87b,false +86415df5-7e47-4637-9e09-aaad9e7628d1,true +8743e69b-17aa-44ff-b8fa-4f582665efc6,false +87b04a97-1d33-4548-aec9-3d2856070702,false +88b16960-bfff-41d0-81e4-9a4630834a00,true +89f260bb-68b4-4dec-91f4-d52e673e0ff7,false +8a1908f7-47cc-4f82-859c-781a193e9901,false +8bc7af32-e5ad-4849-ba4d-b446db833ab4,true +8be81657-8ba6-4ec5-8ff9-4809367096ea,false +8dc2ce26-aec7-43c0-9771-350af4257ad8,true +8e9e848c-0547-4f2c-8b5f-e33d79bbed67,false +8f76d729-9f74-4cfd-be2a-8b033b568e18,true +900ce9fe-02d3-476b-902a-9467767ecdcf,false +922e2443-9cc5-421f-8310-9cd23f6e9f2d,false +96b2282d-1384-4cfb-9958-f009fb501626,true +97b42d67-8691-433d-8a31-dada076162ae,true +984067fc-3dfc-47b7-8ee0-4d9b27d43860,true +990c21ab-433b-4920-873e-f9dfc7103d9d,false +99fc3558-79f3-4d14-9aa1-ff63f621ecd9,false +9b2eb632-6f1a-4e97-912b-3c8378a1b11c,true +9b4f52b0-51fa-4ea6-b02c-2ed8478066d3,true +9bc516d4-7c82-4340-a90c-bb493f96fbe4,false +9e0b8dc3-65c4-490d-97c0-d5e3db944de5,false +9ed8703e-3b40-424c-9e98-b3b404051f99,false +a1228f20-7c50-4d3a-b88c-2d277ca79d79,false +a221198d-000e-497b-8b70-bb4d37f7bbe1,true +a2c3ee1d-ec35-4b26-9bab-12de3f47d604,false +a6790cbd-8349-432e-a976-5dfc07451203,true +a6b084fb-ada7-480a-9adc-f180d4eb1e2a,false +a6cb423a-7571-46df-83e2-ccc6820adb36,true +a89c0c42-c19f-4d7f-a15e-d04e043c92f6,true +acb2329d-b1c3-45c3-ad28-91a09aa521e1,false +add6d754-a075-4693-a33a-c061c9a368ff,true +ae7ddaef-4911-418c-8401-d978617e145b,true +ae9e663e-8c15-43c9-8a88-52b61cbf07a9,false +b00df815-7528-4967-bfe2-311130d91c21,true +b07fb98d-2da4-423a-83b4-7a673de93096,true +b0d337c5-3555-4817-ba59-4ceea5ad8a91,true +b1ccd55a-6b88-4240-b20c-ca893f36e23b,false +b1d4dc41-07ae-44db-ae17-1df86c5a62cf,true +b5d06aa6-b5c6-4782-a0f8-dbf4121e1baa,false +b6900c21-d310-4fb4-8b8f-176a09c91989,true +b7772635-1801-48e4-a442-f4aaa6860544,true +b870d2cb-ea6a-4b67-8de5-ce1224a18bf2,false +bac374c0-50e8-4a5c-947b-82a8e9a747a9,false +baf48725-f0f9-4357-a71d-b1104910deae,true +bb1a7816-4798-4f58-9ce4-c5bd3af682a8,false +bbd0f936-0cb9-4cc9-858c-2926b2b1c1eb,false +bf895c48-1d52-4780-8e9a-532d69356d28,true +bfc93cb7-e53a-49f7-b86e-81d6a13cfdea,true +c06a9f9b-90fa-4bd6-9581-0c6950f03e4a,true +c3ec1f2e-1411-41e4-8370-acd5a8f68cf4,false +c57d51a7-45de-41b9-92fc-efd0634effaf,false +c5c6eed5-aec6-4b8a-b689-adbb219c2267,false +c892b1d7-7485-407d-8883-54fb7fecebc1,false +c90aae7e-5704-4e13-8794-41ab377193bd,false +ca3e0486-fd6b-4a13-a741-3a47760b412d,false +ca7f269d-3794-4994-8c46-d8e9f2aa6378,false +cc56af6d-25c6-480e-9767-da02a55551da,false +cdb8493c-e3b0-447f-b16b-e58dd64660f3,false +cf67a8d4-48e2-4dc9-a521-6aabcff0330b,true +cfcbe34a-edc5-4442-bc05-5927fa00336b,true +d05461d6-13bf-402c-890d-db6404933f7c,false +d080c116-681a-494a-a928-45924109b49d,false +d535b213-2abc-438f-aa6a-c06476d60b09,false +d6988116-3c63-44f8-9f94-9e9d51cc5a37,true +d79728e1-8834-4f4a-8edc-ce18aca18be6,true +d7a48a26-7731-4046-bf22-78f0b8084eb9,false +d90676d2-ce74-4867-a00f-6dbffa7c00be,true +d9e7dd88-2d7c-4e48-8cf9-9e33f729b7c2,false +db79f75d-af3c-4f82-b4e5-9b5d26598eef,false +def5fd23-2b74-4c64-85e9-fac27e626bb7,true +dfe38e2d-23d5-45c6-86bd-f83fe3b6a1d8,false +e0dfbc08-45ad-4a81-b648-7e65c066f673,true +e1040d58-53bc-4467-8101-ca53b772cede,true +e1c2ad9f-6f61-4f16-805a-2f405b63659a,true +e2387a81-5ff5-4daf-b2e8-881998d0d917,true +e26ae29a-213a-4f79-98c2-cc81cf2f451d,false +e363eb67-64c7-440f-93fd-5c8787c69a85,false +e4358f28-62a8-4e06-b2bd-cb00d3310349,false +e7f68d35-ae55-4fa4-b0be-0672a5a7186a,false +e8c41168-a093-40eb-8baa-6802d24f554a,false +e8eba267-fecb-477e-9625-771fbcfc3cba,true +ede7745a-8f3e-4e20-9f16-33756be7ab6c,false +ee24bf89-81a3-4259-a382-86917f3829d4,false +f1293c5e-c997-4ba9-8bfd-438e7a840dc8,false +f18b08bd-40da-43c1-87ec-4ba23542635f,false +f19eefca-c1ad-4860-bdda-4674a631d464,false +f28931e5-1f35-4f9f-a02e-78398735ea67,false +f29330d0-5b32-4f75-b558-a1c7f05c0b4a,false +f2c9ca81-8b1e-4873-bd8a-1a6e1411193c,true +f3aa491a-4dbd-4436-b674-18b2177dcf18,false +f3c2f842-6aa3-42c9-a0f3-895c40456868,true +f3eb0c38-2571-44e7-be39-732eb52cf1df,true +f6d3ff7b-f6e1-416a-bb05-73dfa8d129b5,true +f77a8561-5adc-452a-ac9a-76273b6a6678,true +f883f2ca-d523-4c9f-86b2-0add137901de,false +fc4d37ec-22b7-4ea0-a34e-dcd4beb2001c,false +fc771883-3bd6-46d9-9626-a130e1b902de,false +fcab9efe-fb05-42d6-b366-ce6db1cc4093,true +fcc916dd-5e4f-4bf7-9aef-c906a36d7301,false +fce20464-ff98-4051-8714-6b9584e52740,true From 589f1f7fedd60c6da20788c0f435933b8aa774ef Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 3 May 2022 20:58:51 +1000 Subject: [PATCH 23/83] Add QuantityParserTest and MathOperatorQuantityTest#volumeArithmetic --- .../datatypes/DecimalCustomCoder.scala | 6 +- .../fhirpath/element/QuantityPath.java | 15 +- .../fhirpath/encoding/QuantityEncoding.java | 14 +- .../fhirpath/literal/QuantityLiteralPath.java | 15 +- .../terminology/ucum/ComparableQuantity.java | 19 +- .../csiro/pathling/terminology/ucum/Ucum.java | 2 +- .../operator/MathOperatorQuantityTest.java | 69 +- .../fhirpath/parser/QuantityParserTest.java | 58 + .../pathling/test/helpers/SparkHelpers.java | 17 +- .../pathling/test/helpers/TestHelpers.java | 2 +- .../lengthObservationComparison.csv | 1502 +++++++++++++++++ .../lengthObservationSubtraction.csv | 1502 +++++++++++++++++ site/docs/fhirpath/data-types.md | 2 +- 13 files changed, 3179 insertions(+), 44 deletions(-) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java create mode 100644 fhir-server/src/test/resources/responses/ParserTest/lengthObservationComparison.csv create mode 100644 fhir-server/src/test/resources/responses/ParserTest/lengthObservationSubtraction.csv diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala index c87adad3ce..c3ba653a25 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala @@ -112,12 +112,12 @@ object DecimalCustomCoder { * For location coordinates 6 decimal digits allow for location precision of 10cm, * so should be sufficient for any medical purpose. * - * So the final type is DECIMAL(26,6) which allows both for 6 decimal places and - * at least 18 digits (regardless if there any decimal digits or not) + * So the final type is DECIMAL(32,6) which allows both for 6 decimal places and 26 digits + * (regardless if there any decimal digits or not) */ val scale: Int = 6 - val precision: Int = 26 + val precision: Int = 32 val decimalType: types.DecimalType = DataTypes.createDecimalType(precision, scale) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 02a1ec607b..addf6d5ad1 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -106,17 +106,18 @@ public static Function buildMathOperation(@Nonnull fina return target -> { final Column sourceQuantity = source.getValueColumn(); final Column sourceComparable = source.getNumericValueColumn(); + final Column sourceContext = source.getNumericContextColumn(); final Column resultColumn = operation.getSparkFunction() .apply(sourceComparable, target.getNumericValueColumn()); final Column resultStruct = struct( - sourceQuantity.getField("id").as("id"), + sourceContext.getField("id").as("id"), resultColumn.as("value"), - sourceQuantity.getField("value_scale").as("value_scale"), - sourceQuantity.getField("comparator").as("comparator"), - sourceQuantity.getField("unit").as("unit"), - sourceQuantity.getField("system").as("system"), - sourceQuantity.getField("code").as("code"), - sourceQuantity.getField("_fid").as("_fid") + sourceContext.getField("value_scale").as("value_scale"), + sourceContext.getField("comparator").as("comparator"), + sourceContext.getField("unit").as("unit"), + sourceContext.getField("system").as("system"), + sourceContext.getField("code").as("code"), + sourceContext.getField("_fid").as("_fid") ); final Column sourceCanonicalizedCode = source.getNumericContextColumn().getField("code"); final Column targetCanonicalizedCode = target.getNumericContextColumn().getField("code"); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 0b46af71e5..bf4ee1e304 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -7,7 +7,6 @@ package au.csiro.pathling.fhirpath.encoding; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; -import au.csiro.pathling.fhirpath.element.DecimalPath; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Optional; @@ -32,9 +31,12 @@ public static Row encode(@Nullable final Quantity quantity) { } final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); - return RowFactory.create(quantity.getId(), quantity.getValue(), - DecimalPath.getDecimalType().scale(), comparator, quantity.getUnit(), quantity.getSystem(), - quantity.getCode(), null /* _fid */); + if (quantity.getValue().scale() > DecimalCustomCoder.scale()) { + quantity.setValue( + quantity.getValue().setScale(DecimalCustomCoder.scale(), RoundingMode.HALF_UP)); + } + return RowFactory.create(quantity.getId(), quantity.getValue(), quantity.getValue().scale(), + comparator, quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */); } @Nonnull @@ -47,7 +49,9 @@ public static Quantity decode(@Nonnull final Row row) { // alongside it. final int scale = row.getInt(2); final BigDecimal value = Optional.ofNullable(row.getDecimal(1)) - .map(bd -> bd.setScale(scale, RoundingMode.HALF_UP)) + .map(bd -> bd.scale() > DecimalCustomCoder.scale() + ? bd.setScale(scale, RoundingMode.HALF_UP) + : bd) .orElse(null); quantity.setValue(value); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 83848dc78a..407d8dd5dd 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -22,6 +22,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.Getter; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; @@ -77,7 +78,7 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, final String unit = StringLiteralPath.fromString(rawUnit, context).getValue() .getValueAsString(); - final String validationResult = ucumService.validate(unit); + @Nullable final String validationResult = ucumService.validate(unit); if (validationResult != null) { throw new InvalidUserInputError( "Invalid UCUM unit provided within Quantity literal (" + fullPath + "): " @@ -85,9 +86,9 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, } final BigDecimal decimalValue = getQuantityValue(value, context); - final String display = ucumService.getCommonDisplay(unit); + @Nullable final String display = ucumService.getCommonDisplay(unit); - return buildLiteralPath(decimalValue, unit, display, context, fhirPath); + return buildLiteralPath(decimalValue, unit, Optional.ofNullable(display), context, fhirPath); } @Nonnull @@ -120,14 +121,14 @@ private static BigDecimal getQuantityValue(final String value, final @Nonnull Fh } @Nonnull - private static QuantityLiteralPath buildLiteralPath(final BigDecimal decimalValue, - final String unit, final String display, final @Nonnull FhirPath context, - final String fhirPath) { + private static QuantityLiteralPath buildLiteralPath(@Nonnull final BigDecimal decimalValue, + @Nonnull final String unit, @Nonnull final Optional display, + final @Nonnull FhirPath context, @Nonnull final String fhirPath) { final Quantity quantity = new Quantity(); quantity.setValue(decimalValue); quantity.setSystem(Ucum.SYSTEM_URI); quantity.setCode(unit); - quantity.setUnit(display); + display.ifPresent(quantity::setUnit); return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity, fhirPath); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index 3b93c4038e..0b15b774ec 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -6,12 +6,14 @@ package au.csiro.pathling.terminology.ucum; +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import au.csiro.pathling.fhirpath.element.DecimalPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.udf.SqlFunction1; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -95,11 +97,24 @@ public Row call(@Nullable final Row row) throws Exception { // Use the UCUM library to get the canonical form of the Quantity. final int maxPrecision = DecimalPath.getDecimalType().precision(); final Decimal value = new Decimal(input.getValue().toPlainString(), maxPrecision); - final Pair canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); + @Nullable final Pair canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); + + // If the canonical form is not complete, we can't compare the quantities. + if (canonical == null || canonical.getCode() == null || canonical.getValue() == null) { + return null; + } + + BigDecimal canonicalizedValue = new BigDecimal(canonical.getValue().asDecimal()); + if (canonicalizedValue.precision() > DecimalCustomCoder.precision()) { + return null; + } else if (canonicalizedValue.scale() > DecimalCustomCoder.scale()) { + canonicalizedValue = canonicalizedValue.setScale(DecimalCustomCoder.scale(), + RoundingMode.HALF_UP); + } // Create a new Quantity object with the canonicalized result. final Quantity result = new Quantity(); - result.setValue(new BigDecimal(canonical.getValue().asDecimal())); + result.setValue(canonicalizedValue); result.setUnit(canonical.getCode()); result.setSystem(Ucum.SYSTEM_URI); result.setCode(canonical.getCode()); diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java index 996e653b12..63e0b37aef 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java @@ -24,7 +24,7 @@ @Profile({"core", "fhir"}) public class Ucum { - public static final String SYSTEM_URI = "http://unitsofmeasure.org/"; + public static final String SYSTEM_URI = "http://unitsofmeasure.org"; @Bean @Nonnull diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java index b4f776627a..c0b24943ff 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java @@ -12,12 +12,14 @@ import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.fhirpath.parser.ParserContext; import au.csiro.pathling.test.builders.DatasetBuilder; import au.csiro.pathling.test.builders.ElementPathBuilder; import au.csiro.pathling.test.builders.ParserContextBuilder; import au.csiro.pathling.test.helpers.TestHelpers; import ca.uhn.fhir.context.FhirContext; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -28,9 +30,11 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.Quantity; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.params.ParameterizedTest; @@ -49,6 +53,9 @@ public class MathOperatorQuantityTest { @Autowired FhirContext fhirContext; + @Autowired + UcumService ucumService; + static final List OPERATORS = List.of("+", "-", "*", "/"); static final String ID_ALIAS = "_abc123"; @@ -100,16 +107,16 @@ Dataset expectedResult(@Nonnull final String operator) { final Row result; switch (operator) { case "+": - result = rowForUcumQuantity(1650.0, "m"); + result = rowForUcumQuantity(new BigDecimal("1650.000000"), "m"); break; case "-": - result = rowForUcumQuantity(-1350.0, "m"); + result = rowForUcumQuantity(new BigDecimal("-1350.000000"), "m"); break; case "*": - result = rowForUcumQuantity(225000.0, "m"); + result = rowForUcumQuantity(new BigDecimal("225000.000000"), "m"); break; case "/": - result = rowForUcumQuantity(0.1, "m"); + result = rowForUcumQuantity(new BigDecimal("0.100000"), "m"); break; default: result = null; @@ -138,19 +145,21 @@ FhirPath buildQuantityExpression(final boolean leftOperand) { .withIdColumn(ID_ALIAS) .withStructTypeColumns(quantityStructType()) .withRow("patient-1", leftOperand - ? rowForUcumQuantity(150.0, "m") - : rowForUcumQuantity(1.5, "km")) + ? rowForUcumQuantity(new BigDecimal("150.0"), "m") + : rowForUcumQuantity(new BigDecimal("1.5"), "km")) .withRow("patient-2", leftOperand - ? rowForUcumQuantity(7.7, "mSv") - : rowForUcumQuantity(1.5, "h")) // Not comparable + ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") + : rowForUcumQuantity(new BigDecimal("1.5"), "h")) + // Not comparable .withRow("patient-3", leftOperand - ? rowForUcumQuantity(7.7, "mSv") - : rowFromQuantity(nonUcumQuantity)) // Not comparable + ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") + : rowFromQuantity(nonUcumQuantity)) + // Not comparable .withRow("patient-4", leftOperand ? null - : rowForUcumQuantity(1.5, "h")) + : rowForUcumQuantity(new BigDecimal("1.5"), "h")) .withRow("patient-5", leftOperand - ? rowForUcumQuantity(7.7, "mSv") + ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") : null) .withRow("patient-6", null) .buildWithStructValue(); @@ -171,4 +180,40 @@ void test(@Nonnull final TestParameters parameters) { assertThat(result).selectOrderedResult().hasRows(parameters.getExpectedResult()); } + /* + * This test requires a very high precision decimal representation, due to these units being + * canonicalized by UCUM to "reciprocal cubic meters". + * See: https://www.wolframalpha.com/input?i=3+mmol%2FL+in+m%5E-3 + */ + @Test + void volumeArithmetic() { + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", rowForUcumQuantity(new BigDecimal("1.0"), "mmol/L")) + .buildWithStructValue(); + final FhirPath right = new ElementPathBuilder(spark) + .expression("valueQuantity") + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(rightDataset) + .idAndValueColumns() + .build(); + final FhirPath left = QuantityLiteralPath.fromUcumString("1.0 'mmol/L'", right, + ucumService); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + final OperatorInput input = new OperatorInput(context, left, right); + final FhirPath result = Operator.getInstance("+").invoke(input); + + final Dataset expectedDataset = new DatasetBuilder(spark) + .withIdColumn() + .withStructTypeColumns(quantityStructType()) + .withRow("patient-1", + rowForUcumQuantity(new BigDecimal("1204427340000000000000000.00"), "m-3")) + .buildWithStructValue(); + assertThat(result).selectOrderedResult().hasRows(expectedDataset); + } + } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java new file mode 100644 index 0000000000..0bca573717 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.parser; + +import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.element.BooleanPath; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import java.util.Collections; +import org.hl7.fhir.r4.model.Enumerations.ResourceType; +import org.junit.jupiter.api.Test; + +public class QuantityParserTest extends ParserTest { + + @Test + void lengthObservationComparison() { + final ResourcePath subjectResource = ResourcePath + .build(fhirContext, database, ResourceType.OBSERVATION, ResourceType.OBSERVATION.toCode(), + true); + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .terminologyClientFactory(terminologyServiceFactory) + .database(database) + .inputContext(subjectResource) + .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) + .build(); + parser = new Parser(parserContext); + + assertThatResultOf("valueQuantity < 1.5 'm'") + .isElementPath(BooleanPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/lengthObservationComparison.csv"); + } + + @Test + void lengthObservationSubtraction() { + final ResourcePath subjectResource = ResourcePath + .build(fhirContext, database, ResourceType.OBSERVATION, ResourceType.OBSERVATION.toCode(), + true); + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .terminologyClientFactory(terminologyServiceFactory) + .database(database) + .inputContext(subjectResource) + .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) + .build(); + parser = new Parser(parserContext); + + assertThatResultOf("valueQuantity > (valueQuantity - 2 'g/dL')") + .isElementPath(BooleanPath.class) + .selectResult() + .saveAllRowsToCsv(spark, + "/Users/gri306/Code/pathling/fhir-server/src/test/resources/responses/ParserTest", + "lengthObservationSubtraction"); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 79467d5bc2..900aed8171 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -12,6 +12,7 @@ import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -41,8 +42,6 @@ @SuppressWarnings("WeakerAccess") public abstract class SparkHelpers { - public static final int DECIMAL_SCALE = 2; - @Nonnull public static IdAndValueColumns getIdAndValueColumns(@Nonnull final Dataset dataset) { return getIdAndValueColumns(dataset, false); @@ -167,15 +166,23 @@ public static Row rowFromQuantity(@Nonnull final Quantity quantity) { final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); return new GenericRowWithSchema( - new Object[]{quantity.getId(), quantity.getValue(), DECIMAL_SCALE, comparator, + new Object[]{quantity.getId(), quantity.getValue(), quantity.getValue().scale(), comparator, quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */}, quantityStructType()); } @Nonnull - public static Row rowForUcumQuantity(@Nonnull final Double value, @Nonnull final String unit) { + public static Row rowForUcumQuantity(@Nonnull final BigDecimal value, + @Nonnull final String unit) { final Quantity quantity = new Quantity(); - quantity.setValue(new BigDecimal(value)); + quantity.setValue(value); + if (quantity.getValue().scale() > DecimalCustomCoder.scale()) { + quantity.setValue( + quantity.getValue().setScale(DecimalCustomCoder.scale(), RoundingMode.HALF_UP)); + } + if (quantity.getValue().precision() > DecimalCustomCoder.precision()) { + throw new AssertionError("Attempt to encode a value with greater than supported precision"); + } quantity.setUnit(unit); quantity.setSystem(TestHelpers.UCUM_URL); quantity.setCode(unit); diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java index 233f0c69c3..a9532587f6 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/TestHelpers.java @@ -43,7 +43,7 @@ public abstract class TestHelpers { public static final String LOINC_URL = "http://loinc.org"; public static final String SNOMED_URL = "http://snomed.info/sct"; - public static final String UCUM_URL = "http://unitsofmeasure.org/"; + public static final String UCUM_URL = "http://unitsofmeasure.org"; public static final String PARQUET_PATH = "src/test/resources/test-data/parquet"; public static final MediaType FHIR_MEDIA_TYPE = new MediaType("application", "fhir+json"); diff --git a/fhir-server/src/test/resources/responses/ParserTest/lengthObservationComparison.csv b/fhir-server/src/test/resources/responses/ParserTest/lengthObservationComparison.csv new file mode 100644 index 0000000000..e7abc33dc5 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/lengthObservationComparison.csv @@ -0,0 +1,1502 @@ +0020b035-7314-4867-9453-7921f158d003,"" +002593d4-d271-46a9-a900-1c498c322988,"" +00e9f82f-f16c-422f-81a3-77fa98958da3,"" +00f02590-e737-42c4-9c73-206cf66b02b7,"" +011f5b11-f944-46ae-9cde-0d802e5c0073,"" +0143f01c-3692-479a-9a72-592260dbbd64,"" +01a4de83-37d0-4612-82eb-2ff2dbc2e02b,"" +01c177be-05bb-4a44-a199-1df387e68d8e,"" +01c17bf0-9240-4149-983e-f0b3ccfb52be,"" +01d57966-1281-42ab-9e05-02d96d146b6e,"" +01e27cc9-c9c0-4c33-82f6-9507394f5f2a,"" +0218f780-899c-4c1b-9167-45e33d02acfb,"" +021bf465-c299-4302-99ca-e29a6d25bda5,"" +022ebe16-f618-4acb-9d99-88e23d686c34,"" +023a9220-3016-4202-8c7d-360d7d3bece9,"" +025aa37b-4ae6-473b-9ae6-c12eece968c5,"" +02741df6-79f0-477c-963c-8b3f8cb4374f,"" +02b87530-10e2-4009-8734-4196642cef1d,"" +02c07924-f5c7-4909-ab49-dec34c051dfc,"" +02c4fa06-3281-46a0-9bb9-c7b99497c91d,false +03365fa5-36d2-41b3-b3d9-1fea9f94c73e,"" +03555f06-db5f-4b8c-aae3-f6d2c547f499,"" +03610d59-94a2-41de-a039-c671241b075a,"" +038ced3d-5233-44c6-9a8c-1f6f9cc4be11,"" +03c7ba3d-8af2-44d1-a3c0-2c56b9fbc7c6,"" +03dadf3e-cdc9-4f63-8a4d-89bae77bd529,false +03e44215-f743-4c4e-8d19-aaf761f54503,"" +0429fff2-2839-45d8-b0bd-1d86aa92f966,true +042bb1e3-e5aa-471e-9e83-bb4a1b90af22,"" +043108b6-98cc-4feb-8ad0-1394cd10a289,"" +04367369-256f-4d47-b1ef-8014109ab46f,"" +04661736-24d8-41f7-9f05-af5682696df0,"" +05048b00-24df-4311-806b-2f9a7cc16380,"" +0531e6b8-2ede-4e12-a91b-cfe7e12e552e,false +054334cd-cfc9-4a1c-acf3-d0824cafd3a5,"" +055824d7-b398-40b2-8fed-bc40552fffce,"" +05643ef2-65e0-4812-b739-4fadfa3c066e,"" +05bed469-760c-439b-9729-0479a6550b76,"" +05c37921-dde8-4b70-869e-82680975ab77,"" +05daccae-2d73-459d-8aea-7ee75966e52b,"" +0624683a-8572-4885-b51d-2197dde395bf,"" +062ab32e-318d-4ff7-81ea-f3be53204988,"" +068d13a5-8b46-4a3b-aca0-ac9ffe70f998,"" +071b684c-70a5-4c93-8289-9630d25496d2,"" +072312b8-c984-4cb4-a4f8-97560368576d,"" +07279076-3064-44aa-b07b-346a7edf6abb,"" +072bd136-7e5d-49a2-b878-aaa2c7fde1f2,"" +0735b01a-c65e-4d59-9016-5e3bed1fcdcf,"" +0785f996-ee5e-46f3-8966-107235de4cde,"" +07906572-63db-4046-8dbf-1ae866664f41,"" +07d441b6-4e92-4d6d-af4b-7f38471fa45e,"" +07dfda29-5378-4a66-ac90-fcb7ec6283ca,false +07f5fcdd-3aa0-4ccd-9122-f9686886dbd4,"" +07f95b0d-656b-4830-ade2-864e58ac5ed6,"" +088cdf48-9645-4dbf-b824-8836d6a83d24,"" +08d37a26-1ec6-4faa-985a-10a9d0ee54a9,"" +092b1b96-4ad3-4716-850e-8056ccd456d8,"" +094a486e-7e50-4717-a0c3-07a456a4cd5f,"" +09541c80-54b4-4bb0-8776-37745a9ac36e,"" +09941e9e-ff5e-44e5-82cd-4c3865e39c01,"" +09ab1f16-7bff-44ec-aed5-8ba400930bbc,"" +0a005143-4645-4ce4-89d5-9d49dca18ec6,"" +0a40e26c-d0ac-4653-a54f-17e23ed9a0ce,"" +0a6c1262-dae3-4ec2-b911-0d1caca68302,"" +0a7887db-203b-4a3a-b5fb-cabe9a886074,"" +0ab96751-edf1-4941-981c-c6bdcba75380,"" +0ae172de-1755-46d4-8729-d3acc96ce82f,"" +0b6ee068-26bf-4f1b-8423-30f1c591861b,"" +0b95920f-7c6d-4b30-89da-51807a6aa557,"" +0bbc7111-589b-460e-a742-20f2131c9c64,"" +0bd78173-c348-41f1-9edd-884e3561ad47,"" +0c4242e4-1c5e-49d3-bf57-3810f8b88d56,"" +0c93a4a8-3b59-41f8-a7a5-22a9691710ab,"" +0c93c94d-b626-40b2-9561-52168c4961ca,"" +0cbe226b-6432-41c9-b3dd-0a14c6d54d7e,"" +0d1af4fe-50f7-4d8e-823f-192f8a08ec8a,"" +0d7a0166-7fb2-4e1a-b1e5-b03a53fb5602,"" +0dbafa74-2f5b-400e-b27e-fddeac8c28ad,"" +0df8e035-a226-425d-9684-a12d0c58b844,"" +0e03654b-d81d-4a62-8f53-b479e8ce3edd,"" +0e11e8ea-dc1b-4e8f-b809-39f0f07c7499,"" +0e8063ca-3601-4741-a2c0-c0381cc6098b,"" +0e92574a-cc09-4564-a2ff-1cdbc89d210c,"" +0e99ca14-e99e-4394-b962-59c66427218d,false +0e9cc169-ca91-4201-a78f-1b0f724507cb,"" +0ed9ee59-d968-443c-b7e4-c3732ec666e3,"" +0edd2d17-3ac8-427c-ae05-40c99b401484,"" +0ee77005-b9a1-4c51-a599-bd233897e431,"" +0eea9efa-148d-41d4-b05a-d39484526333,"" +0f13292a-a654-4473-b78e-c8f1b3328ff2,"" +0f67bae5-3bbc-4c7d-84f4-34583758335d,"" +0f68a16f-1c70-4d73-a9b2-fb384585526b,"" +0f7373d2-3052-4c98-9c04-7074250f44cc,"" +1016a9db-50e6-4c17-9e93-18a97b67269e,"" +1037da65-9451-48d1-a35d-bfcae76c2c60,"" +10a7d4aa-af9d-4d03-8d1b-bb53ed123f62,"" +10ad7a5d-e26a-4d4b-b063-239af96b0e87,"" +10b85b80-f6fb-4473-a3a6-266f610dcbc6,"" +10cd7171-38a4-4276-b911-bcc8092eff7c,"" +10fef9b1-6bdf-4eef-b6b5-e12504e8fc7d,"" +1104d152-e95e-4a6a-8225-ab200e4530b4,"" +11148ea5-efc3-47ad-8672-799331f8e491,"" +114ead73-4920-48fd-847d-f9aa1a99cab3,"" +11597b54-9453-4ddc-9171-2be17df7a511,"" +1167e4a5-ce86-48b0-8b96-6d70125e2f9e,"" +11a0c4d2-8908-4c0f-8e51-bf4be1605441,"" +11af3d82-bb56-464a-9203-8714c270f4a5,"" +11cc59ea-52cb-41e1-97e4-7ed651037442,"" +12254e32-b86c-45dd-93b0-c8cbee619a14,"" +12257920-12cf-4945-a72f-80551181d7f1,"" +1266203f-94c9-43ac-af02-a0b8db8a93dc,"" +12689eb6-84ee-4d28-9c65-3f3e305266f3,false +127b14bb-027d-4380-932e-ea34c76592d4,"" +128cf2ff-4f97-4039-983b-72e2dd3267aa,"" +1333fbed-c09d-4622-b22c-c67dd32e4583,"" +133a1365-e43c-4d92-a6f8-8c3947b13b65,"" +1359fe32-3275-4c1b-ba16-9dc903334044,"" +137d6ce5-93fb-4201-8bf9-0e97e9595b77,"" +13bf741f-7ad6-4a14-908f-cbfe18651381,"" +13f4a39d-ec9c-43b4-958e-be28a97972e2,"" +143099ce-8738-4ccf-85fe-63e54943c7f6,"" +146000e6-9926-43da-9322-de3bf31e8142,"" +147add3d-eba6-4cea-bbe9-823b44794b9a,"" +14a9ad6b-b7bf-4cdd-972c-bda91e5066bd,"" +14afc7c1-89c8-4740-a932-a76d579e98c3,"" +151df048-c17f-4a51-83a9-3de5f99be125,"" +1520180e-fb4a-4df6-80be-0b3d559c1049,"" +15268027-ae33-4f5e-9c2c-76327ccdeb44,"" +155d0003-f37b-45de-b34c-d611c2a2cf35,"" +15821e91-ec03-4e76-a909-d2f2a14c3219,"" +159c1f96-038f-4722-9e36-98f4dfdb6b66,"" +159c3713-0d0f-499f-8ff6-ecddd44b958a,"" +15d9442a-2018-4390-beef-55fd90cf6ea3,"" +15e70015-e207-49dd-93a3-7c0ea5cc456f,"" +1631e3a1-0eb3-4d16-a7e5-b0d8a8a9e630,"" +16af45f4-69d1-4190-9bca-b309c17bd865,"" +16be31b9-a434-46e2-a481-459c5584c907,false +170e5f14-b29a-4aa8-ba71-4bf381a38d1c,"" +17505b4c-c5e8-4acd-9d10-fb001190a8c5,"" +1795e970-0ce6-4d90-b198-32567c690869,"" +17d74e33-f6c3-4b19-8708-4110a2f55245,"" +18a57e28-9c23-4e73-9d90-3694f2e373f2,"" +18c6a217-c947-4e97-a764-c43f33a409fb,"" +18e59ddb-b9b9-401d-9239-0d2cb81936c8,"" +18ee3e63-a8a2-4004-a313-3e332af8a173,"" +190f22f4-5fe5-47e5-974a-fbbe63a676c3,"" +191232f1-58d7-4c4a-93a9-10aee15e5cea,"" +1913a538-60f6-4510-9a76-2466896d2a27,"" +193a64c2-63ea-4677-9302-b56e3f0531ac,"" +19a30860-10da-4a1b-abdf-0b03d7b8f370,"" +19b01508-db72-4c22-a5b4-4e015a3207f9,"" +19c965ca-c1ae-4eec-816f-73903fe43aa8,false +19d27e58-26a0-48ca-a199-88d48894da5c,"" +1a3ad6e3-cb5e-483b-babc-ba7d34ffacf7,"" +1a7b0249-5aa4-4b0b-adb8-2c958e137d54,"" +1ac6d3a1-d1a6-4b0c-b405-0aaaaf37c7db,"" +1adf64f4-1fb4-47f9-a8a3-0ebeb3e7f414,"" +1af4a596-d0c3-4f7c-a217-b83a0755c3b7,"" +1b091e60-4c4b-4fb4-b9b8-ee53bccc9b52,"" +1b251a42-a224-4a0f-9d36-44ecc8902d08,"" +1b29b772-b873-447b-8aa0-5dd5e9ee1b80,"" +1b614d48-0af0-4d09-a6af-4f3af065da14,"" +1b785bb6-a40c-48e6-a464-f8022ea3cdcb,false +1bb3d0c2-c6de-42ec-bd60-edc8e3ccd092,"" +1bdd0326-987d-445b-9514-2ea30d948093,"" +1c01b494-d0c1-4a99-b110-9de392bbcf69,"" +1c120e92-0385-4120-ac83-1b42191597b8,"" +1c32d0e8-7312-4821-aede-6745a83ea45b,"" +1c51f4a8-14a6-48ad-9821-c6408dd0cd33,"" +1c52440e-0063-4170-9cff-a3caaf7c63fb,"" +1c6dd7c8-4942-498d-a369-65fac6a837d6,"" +1c79c271-a128-41be-9537-1d744f4971a8,"" +1ce6a20a-2413-470f-86ca-2dacd8811e72,"" +1d9b87ab-8b41-4184-9519-fa9822f48aa1,"" +1da7f725-2fee-4f33-9a89-96253d2449fe,"" +1dcdb77f-b0e5-4b2f-ad4e-d446a6ffa079,"" +1ddc3c30-fccc-4e5d-a9b2-e09ac4f64dc0,"" +1e2bcc56-3121-4b1f-b9c7-7e5503b867db,"" +1e5b7df3-5e81-41c5-b096-d9b5409268ae,"" +1e75c4d7-b570-4339-8c0a-06bdfc9bb485,"" +1e84c68a-4e0e-4980-adea-e46ae2506896,"" +1ea31cfd-f340-4c68-a547-b76365abd1bd,false +1f170771-0d08-45db-a38d-da3c539b46b0,"" +1f182acd-6bb8-4031-aa86-2f3dd83b4cb7,"" +1f3e4d9e-1fec-4a71-98fd-4912ab64d3d2,"" +1f6b8ac1-b12c-4eb1-a158-25b15965294c,"" +1f8ad9f7-f796-4900-b10b-365fe442f2e1,"" +1fb81c48-c8d0-44e1-9b9b-df53ad7d9456,"" +1fcab6a5-85c8-41b7-a820-fe6e2d84eaba,"" +2031d9a4-3e03-47a2-ab74-6fe7f2fa4ba1,"" +20d47a7e-f7f1-475b-a9b5-6192f2fce0ca,"" +20ee5488-a11f-418a-86a9-cb981c2fdae5,"" +2123f57e-0815-49e1-aa34-994270134f3a,"" +2160f03b-4276-436a-8278-300bedd07f7b,"" +21936507-6a38-4d80-807d-0d7cb5ea0311,"" +21d2fab0-5092-411b-b752-4859134a7aab,"" +223cae66-3049-48bf-be7b-cf027c415743,"" +22440788-7e64-452d-9025-9a25e4d2be60,"" +2245208e-d17e-4bb7-9bd7-ac75ac0ba134,"" +22462877-9663-4c06-bb5e-040d76bb144f,"" +2253a94d-e511-42bf-a17c-599ae2d5cf0f,"" +226d433b-9abf-4e98-bf0e-47cb3a37b41c,"" +227d84f4-ec92-464a-b3de-f596866ce825,"" +228b1b4e-f5e7-4e33-91fc-b7cbe6979102,"" +22b95566-7a4c-47b1-92af-7e0ecbfee6fa,"" +230b746a-26de-434a-9774-54db1ad98a7c,"" +2324dacf-f221-4edf-bdf6-924449cc916c,"" +2381d397-0068-4d6a-88e1-a0257d81a323,"" +23831569-6aea-4071-820e-6eb5440acc1f,"" +23b4fe3f-62c7-4439-9364-247ceb9251c1,"" +23f91a73-1b46-414b-becb-918e84177886,"" +2439d9a3-6c58-4ced-a82e-27908ed3df94,"" +2450caca-d74c-4610-ac24-1fae13801eb7,"" +24d8f04b-f035-4a9b-b03f-75be9e244ceb,"" +25085ba1-43d7-4a60-82ab-e15c477313b0,"" +251e7497-4a55-42fc-8e6a-32febded738f,"" +2534b076-583b-4872-84e3-bc96bdee5ef0,"" +2583c0bf-6a7e-4100-a5c1-d249de6ec56d,"" +25da1be8-51c6-41fd-b48d-5227e804ebfa,"" +25e1a428-ab0b-40f5-ac25-1e9899458ec7,"" +2606b58b-a924-4128-b3a4-5e4b91deb692,"" +261583fb-e2d0-44c0-a677-00bd89a3450b,"" +26344e09-20c8-4405-b674-49638169a7a0,"" +266c347b-fa2d-4cc1-a2fb-fe1ba20fbc2e,"" +266cf373-c41a-4cc3-8600-c165a4349a6a,"" +26b4001f-2917-4bf5-bcbf-1ce9dc258171,"" +26fb6882-7f3e-44da-9100-58057c9aeec8,"" +277e09d4-cb89-4748-b9da-39bee94df8b3,"" +27a75a54-d318-4026-bd63-c80f73899376,"" +2803a9d2-2555-4bc6-9718-7a9dcabd6ce4,"" +2803acc2-63bf-49a3-b4fb-0dd0742fb816,"" +281d5649-179d-4c8c-9a0a-d1159a5df8f1,"" +28503262-5aab-44eb-bf51-6fb7534c25ab,"" +2866a0de-36ab-4ec0-a1e1-cce5c697ef85,false +286decfc-83d1-42e8-9a2f-880f145c71b3,"" +2875fd27-8c5b-46ec-91f0-ef3a7a7727b3,"" +28aa1696-dde7-40ff-bb7a-6c5b614405dd,"" +28e8966e-ba9b-4499-bc9c-7f51222e15aa,true +290440da-118e-4e8b-8d0e-d50043c24eac,"" +292aa324-379d-4b82-9750-4c8e23587f92,"" +29506f88-302c-4264-81b7-cf762de48615,"" +29546a71-e5a0-47ae-96da-421275c2cce1,"" +29676c08-88c7-4b0c-82c9-5f6d7dd51f68,"" +2974e695-f2e6-477f-91e4-da8eb654bf24,"" +298120fe-5dbb-4d99-902a-3a6783fc5b6b,"" +298e5099-df2f-4381-8259-b10d0b8c0570,"" +299c851f-3566-405f-a217-84c0703d859c,"" +29b17fc8-8e80-4d7d-93dc-bdba1f2cfb9d,"" +29bfab6a-d760-449c-9198-38e8c63574f2,"" +29d4d79f-8a85-490c-b52d-aaf3bcd9a100,"" +29e54cc5-b484-4818-9c3e-08074812dc7f,"" +2a0d7eba-2dbf-4566-af0a-c16161a8b189,"" +2a133e8f-4dd9-49fd-9e3f-5cf122294091,"" +2a67b2c1-18cc-45a0-ad53-825c73a3aa15,"" +2a71ca62-7f39-4c9b-b3de-6dfcf299298a,"" +2a7f0959-9d50-4ccd-88e1-6635f2502905,"" +2af24d01-6215-4c7b-88b4-6cee45a156f3,"" +2b265cf1-0763-4922-a403-e6f644ed95ba,"" +2b41d0b5-97d4-4a80-bf65-b62ac0cb530a,"" +2b4fd497-9f28-4c87-9b13-64f0d00c0cc3,"" +2b54e31d-4af9-4071-ad49-60cae644ba39,"" +2b60fba0-0143-4f31-8e4c-27236c7f41d8,"" +2b927027-35f1-488b-bb5b-127a8985cc5e,"" +2bc92150-c490-44da-a925-ebddec5cd953,"" +2be63080-a881-43df-95b5-e371df93c6bb,"" +2c018134-14c6-425d-8ab4-bc3974402b05,"" +2c3b614e-7860-47ad-9320-11c5f45d27b0,"" +2d358911-c5eb-480a-a9e6-a6fab52ad7c2,"" +2d45c298-d386-43ab-b3e0-ba07dc527a68,"" +2d6cb13b-fdfe-4219-bc13-7de385998bcc,"" +2dbd4567-e6aa-4e1a-996b-af8384f66b37,"" +2df38530-e49e-4ff9-93a3-816de29d9bfb,false +2df89473-f746-4086-a2f3-8ee2d4f63271,"" +2e0bc41c-fdfa-42ab-abdb-28f13cf2ca20,"" +2e13cb39-148f-4299-b74a-0a8e10092e9b,"" +2e45b9b5-8d45-4994-aaba-1fab72c77c5d,"" +2e6b11f5-cede-4152-bfe9-523b030a2bdf,"" +2e8c312f-53b4-4cd3-b6da-653c34945d76,"" +2ea2af0e-343f-47af-b0fe-6dea108836be,"" +2eb09ab4-c554-4536-a3ce-2979cfb988c6,"" +2edee289-e99a-4a53-a067-f83d29db4c2a,"" +2ee72185-ca8f-40e3-b82d-30408dc8c6f2,false +2ef91918-bdb6-4a6e-9c39-41d8c513b552,"" +2f1bbdc4-beaa-4d49-8791-6548094c7291,"" +2f24078d-9646-40fb-908d-2628d555dcb1,"" +2f260daf-8fb3-4fb9-a435-670ec8f0180c,"" +2f30832d-e6b8-4946-8cf4-0d8da5c359c0,"" +2f33b981-f22f-4983-b347-f81d7d72c50a,"" +2f3f7f7a-ba31-4c1e-8a23-c7d1eb1f255d,"" +2f5d7d1b-847e-430c-988a-40028f9f595e,"" +2f8b35f6-2b55-4ff0-8d79-5de7805d6da1,"" +2fc9f8c2-4a1e-4e8d-8d91-a6c3b3937e8a,"" +2fcfe30d-dc47-4eaf-9df0-7943a2c37fc5,"" +2feac451-2301-4ed5-8bca-baef62790656,false +2febeae3-eb61-4743-8ff9-42547a77a40e,"" +3006707b-dcca-41ce-b138-5d62e6b56984,"" +30371578-4ad0-498b-a491-4e194c7466fd,"" +305d5449-57c6-4362-83cf-55b4dae133de,"" +30ba9f80-e595-460a-abf2-b8f46a62e606,"" +31504825-77aa-4a0f-a634-ca806bd886d3,"" +3169cf91-33f7-4b7e-b49f-d4b884760685,"" +319dfae4-0305-4520-8ad1-9679383cb74f,"" +31acc736-076e-426d-8a32-426d91bf6511,"" +31e26c46-af6e-43a2-8e27-a8342f53e817,"" +31fd2352-7eb4-4ce8-9dfb-b3d3a1410c82,"" +322637fc-2d75-4a8f-89a6-a4c963869112,"" +32596eae-6236-4b3a-99fd-e50856971f59,"" +3286c996-861f-4fe3-90a8-86ec826cdf9a,"" +32bed860-5df5-4bbb-8bd1-c95c016561ee,"" +32d92e2f-d02b-4006-bf16-d1c535948e80,"" +32f3d9fa-6c64-49e8-a6dc-fb51679753e5,"" +330fde18-0a86-4e67-850d-fe1c8b376dab,false +3322da98-67e0-445f-ba1f-225eb7d297a3,"" +33333f69-f8a6-4def-a5f2-bab5a26bf054,"" +33826788-b600-476b-9843-17fbee2cd849,"" +33a0b9dd-c88a-46cd-ba8f-af3d4d6cfc27,"" +33ac0c39-1552-408b-97ea-a90c9187d61e,"" +33b22ef7-01a8-4e8f-8b5d-9aef37e46896,"" +33be0567-5c42-451b-bd71-c09f7de30524,"" +340dbb93-0888-4f47-b18d-0b86d3e40248,"" +341634fb-2646-4c69-b57d-b5f00ff4c83d,"" +3420a3d5-d5c7-402b-a4c5-02e7198c5b7e,"" +342699a7-cb58-4a9f-bb79-306483dfa652,"" +34335eed-f6c0-4e05-ba6b-1ecaa1fc121c,"" +346734fc-4798-48e3-8880-a603d52e9ca9,"" +346a282a-439f-48ac-875a-cc34643d70cc,"" +34a83c04-2ea0-4638-ab6a-0a4bb9a3ad4a,"" +34e44ce8-26fe-48c3-8fe8-5cd8f260aee0,"" +34ea288c-f987-476a-a8ed-0ab2c1b92833,false +3590f230-1be0-4c7b-afc6-5bffec9933ad,"" +35cf6113-d23a-476c-b18d-f86fbfb3ea2d,"" +36160c58-889f-4543-bf04-dd29cac9ce19,"" +361688f0-e909-4c9a-b5d8-10727c9a18fd,"" +365ca17e-d5b1-41d2-982e-3ca7afdc0a0d,"" +3669a209-82a0-4f9e-8026-43cebad7f3ab,"" +366fa6b3-9595-4624-ae30-20257ef4885a,"" +36957f27-97d1-483a-a8ea-bd5b1484ad12,"" +36c9ce76-e047-487a-837c-faffdb2676de,false +36cc6703-7757-4c6b-91a0-a43055c19bb6,"" +3732940c-2fa8-4301-942a-e58fd8d73421,"" +373609ea-276f-4b96-9c94-d96ad532246b,"" +37770a6d-bb26-4499-a6e6-2eb399a8ad07,"" +3793c189-6e61-4f1f-bb19-a01a306267d7,"" +379779f8-dbce-4dfd-b18f-ce0485b0d6aa,"" +37e1570f-2c29-418a-acee-6688b6ab4279,"" +37ea3114-1b65-4c47-9e65-4cbcf339f8c5,"" +37fa89b6-2f6e-4985-b733-1ee4b2fd59e2,"" +381dcabc-52cf-42e9-9624-d4e1270518a9,"" +383aa480-8db2-49ed-9542-c738af90f344,"" +38898dca-756f-4267-aa1d-1fe52a6e4749,"" +389079a9-b902-4212-a8c1-7187e818e016,"" +389a8f10-ad75-4d27-9d11-2c4ba0dc55a6,"" +38c329c2-4078-450f-ae85-e411b80f30ce,"" +38c7c98e-0372-4892-aea0-ba21ed298958,"" +38d59ef7-b207-4d55-b74a-a1da3d0c10df,"" +39d3bc28-0ce7-4d1d-9adc-bb9933568809,"" +39d7db36-7baf-43d7-8a0d-970ae00456d0,"" +3a1941c5-c6bd-45b7-b306-d2a1a51c5eab,"" +3a8a0b91-a492-42de-9b64-fc552476d8ff,"" +3aacaf9d-af9b-4220-a836-1d0c3cfd5245,"" +3ad5553b-58b8-46c9-8023-e1acb85be08c,"" +3b4bf6ba-0f27-4228-b6e1-c16dbf82e5cb,"" +3b7c654f-9a67-4402-a571-a5b0a61559ab,"" +3bae6ece-f05c-4e3d-81a7-64f10f907bb2,"" +3bb8d5a9-87a6-4155-8f3a-7a0c841672c1,"" +3bbe3af0-0f3c-41ee-9c41-70d89b411852,"" +3be31b98-5218-44c6-aa92-1e054793d9df,"" +3c496888-2ead-4d5f-afa4-8ab39b57ca03,"" +3c504a58-92fd-4b2f-a308-21a9588b1850,false +3c8d926d-3156-464b-b385-192257411798,"" +3d2c40b9-2ef8-46d7-8215-bb898170902f,"" +3d87abf4-a569-49b5-962b-32c11816a53f,"" +3dac5f75-f71c-4092-b87a-7b8434af633f,"" +3db4a850-ef45-4fa2-8eb1-b1cba4477d8d,"" +3dc0190f-39b2-4a1f-87d9-1ef41a73fa75,"" +3e4206fe-8856-46be-98af-d6fecd52db92,"" +3e58093d-c387-4a07-85c2-41a519d0c398,"" +3ebd1cdb-5546-4dea-ba79-a88aef933640,"" +3edb8e33-d674-4c25-aa75-7303da0229b9,"" +3f5c46c8-4a8a-4575-b169-6bfd5d752d98,"" +3f771434-cfed-4f34-9671-0ef302b5f9e7,false +40100d97-ad9a-459b-b044-66568d36c0bd,"" +408159a9-8c38-4cb3-9488-d83331f1762d,"" +40a1dd3f-6ffd-4463-bf71-42057ee61689,"" +40a75d7a-909f-4657-9ac1-5b56df539129,"" +40eec0cb-8f50-4f61-92a4-7682c48511f9,"" +40f3ae1e-4855-4d78-9fe9-c8348058088e,"" +4115b0b4-f738-405b-9351-ae11ba4bd07a,"" +411cbadd-c2d1-4fb3-8ddb-d8bcea3aa176,"" +41496990-adc5-44d1-96b8-15010bdecc89,"" +41a3cf00-0965-4c17-bfb5-247a7da43699,"" +41a86c21-2ea1-436a-86a3-9173c7d95d9c,"" +41de6042-e95f-4bca-ba69-f55293248f7c,"" +41ecc0b6-ae98-40e0-a836-9d26070e248c,false +42187018-d27b-4831-913c-575d07e33c54,"" +421e31e1-5ffb-4c6e-957c-728b6218d6da,"" +42350e13-ab61-49ef-aea1-802f6b7ea955,"" +423800b8-5a89-4eec-a12f-451c533fac50,"" +42895609-81ce-4cef-81db-069efe9a207d,"" +429c5342-c5ae-4c55-98aa-e4f7bf0b4670,"" +42ac25e5-8828-49bf-bb5c-a6001966c6ff,"" +42c5e47c-ede1-48eb-bc82-2ed835d67495,"" +42fe77c3-239a-4afe-ad76-8c221b543e32,"" +43179e01-e94b-4f83-bb60-60f181388048,"" +432bf1ce-0e8b-4621-b998-819fee53f80a,"" +43522eea-96c7-41f6-ad41-1c190cece6aa,"" +4358ee55-cc06-46a0-8b44-6b89d09c380f,true +4380d762-baa4-4c30-905c-59b33dbd793f,"" +439dfa4f-917f-40c5-9313-0d5dcf7ecf2f,"" +43bdfaaa-7474-4503-ad93-730d777e6db8,"" +4416e165-8ac3-43e0-a9e4-6c3e789ffdd2,"" +44233ad5-4eb9-4709-9c6c-e7797d9c9f1d,"" +44237e1a-31ca-4635-9ff3-e3c71f587972,"" +4442a5e3-5793-43ae-92ab-f2d87a70cc0b,"" +446f7fac-2202-4149-9860-fe48335349b3,"" +44713d16-f673-456a-881b-b0526ee36651,"" +44b741ee-bdb2-4732-b484-e435854de348,false +44dbff22-7cac-4f70-886f-b8ac2dea7130,"" +44f714c0-88de-43a6-b69a-4d62d8fee57e,"" +4522ef75-31ae-43e5-8e5a-5caf5c83fb35,"" +453cd2e7-723b-4e1d-b414-f46f9152edcf,"" +45467cbe-cc21-4822-94f5-82e1203056ad,"" +4546f579-74ab-445c-8619-431f60a6efdd,"" +455f8ebf-b074-4035-915c-91cd15380d72,"" +459f6da9-0eb8-4b26-85b9-a8f63c587fab,false +45a306de-3731-40c2-8bd5-2c0fee3b9f99,"" +45d45783-5a68-4c06-9b2f-88b7ea1f8347,"" +461bea3d-934e-46fa-a112-9a761140372a,"" +4672ebd9-08d2-43fc-ba92-ce13b587af3b,"" +4675fa7c-6ac8-4bef-8756-88f757370a10,"" +4680ae70-6d17-40fe-b55a-c3152ce8b246,"" +4687b275-d990-41cd-a146-2715079a8ecd,"" +47014b4d-07f1-4bef-9792-82c027525c0f,"" +473bf60d-dab0-4531-9903-95dc6680a660,"" +475e95a3-6d57-40f3-a0af-9daf1b35618b,"" +478fbabf-520e-48a5-b3a1-a3e535d64345,"" +47971ac1-1e72-4788-ba6f-bea608e59076,"" +47f066d8-abf5-4064-84fd-dfb05649e0fc,"" +4808c4d5-7733-4601-8401-632bf5924c06,"" +484a19b7-9816-4a66-8f0b-51cb5f146818,true +48693756-94c2-434e-ad88-e573e5278a6e,"" +48fae21d-0ac1-4e4a-93b7-ac4ba7c2a606,"" +49088daf-d19c-4bab-9084-c67f0fba2c90,"" +491118c7-998f-4d45-8b58-da724ac011e1,"" +4925b797-e285-4d02-9aed-dfac19b0875b,"" +492f6c9e-3e1c-4043-aa08-03c610da33a7,"" +49466fee-df56-4b9f-898f-96b65318d833,"" +495bdeba-9ab9-4756-b4e0-a4aa8b039a02,"" +496f0696-cee8-425a-81d8-1df8fd17e4bd,"" +49cce532-3ba4-4db6-b518-ea934588bfae,"" +49ee8b11-a7c9-455e-9a9b-835dd617e861,"" +49f41136-fc63-48a8-8ee2-b7536d3bf0e7,"" +4a6eb9f9-49a5-4d36-85dd-78512fbe593a,"" +4acf651e-af55-447d-bf0d-9adaef4cf6bf,"" +4ae0f855-534f-41dc-a4b0-879c771e3379,"" +4aeca194-6d0d-48ad-8fd4-6aea3be783fd,"" +4b322f4a-fa06-4509-9783-dbbf12960b77,"" +4b68a6bb-4fca-45ae-bfb4-b814c67bdac7,"" +4b86609f-db97-4a0b-9860-3ce519072521,"" +4bbf3839-bd13-406d-9f60-d9eb9daa4dcd,"" +4bd9e2b4-e195-4a83-ba1c-478e66a7bb5f,"" +4be334e3-94a8-4223-846e-39b5877e9ff7,"" +4be59cbe-d89e-4abc-aca4-410a7e8cf580,"" +4be72d80-9418-44de-ab47-e8f2f2b9f2e4,false +4c9a28de-ab9a-4e69-9ca6-375512c2f149,"" +4ca8a902-7ac4-4634-b1df-456175c39d14,"" +4cef34bd-c677-4647-b940-840a6868cb9c,"" +4d0e9887-c81c-4ea0-9026-41ceb9ec9a51,"" +4d45564e-e14c-4289-a637-06193bfcef2e,"" +4d79c0f1-72b2-4e94-addb-9adce3f49c2f,"" +4dad93cd-b188-4750-bdb0-4a5501568d81,"" +4db059c5-c4d2-440a-84ac-360312e3c590,"" +4dba5ae8-2d30-479e-b831-80d39c396280,"" +4de45bfc-2a41-4e0d-a0e7-c6caf6ca0f24,"" +4df7ae55-1947-4d89-b51a-496b34c4250b,"" +4dfadd90-3f14-4ee9-abbd-a348a5d6a7d9,"" +4e0d77a5-782a-4b03-8867-f51a9704368f,"" +4e74bab4-e328-447e-84d9-a29bb7fa1548,"" +4e8a0b34-71b8-4da8-80f9-f015782b182c,"" +4e9a316e-406b-496b-acf5-6eec80f25ee7,"" +4eab3a46-0ea6-431a-8c65-5b5bdee521cc,"" +4ec34b7a-8389-4d01-bca5-06e678089d41,"" +4f147e12-0e70-4ddb-b4e9-4ea25361f290,"" +4f18952b-be4a-4d77-ad4a-05276a6bce1d,false +4f3f5836-8683-49cb-b5e8-a9881796a202,"" +4f62ec59-e478-40ef-8617-e6ed599866f1,"" +4fe2d379-ba65-4ee2-a584-f0bc63a8523e,"" +5005cfbc-b57d-442d-976b-34164aeb1977,"" +504859eb-2ed0-4184-99f1-79f436fcd692,"" +5085d62c-dc5e-4f25-8f0c-700e57bb71fb,"" +508de40d-3c1a-4bfc-8713-11dbae730004,"" +509d7086-3120-4f61-bc92-4ab924f30ffa,"" +50a9341a-67f3-4d18-9118-373db23eea3c,"" +50b76e4f-6575-47f6-9e15-a96b58c492f3,"" +50dbd55a-e4db-406d-943e-7624d96c554b,"" +50fbe67f-8d29-41a7-8cc6-3df916cd922f,"" +5105a245-dc96-483d-9a0e-52b39e664d27,"" +5106bb8d-43a4-4783-97c9-42d707990a2c,false +51164c4a-fa5e-4246-80e4-a01e1d7707d6,"" +512a1f5b-9f83-4420-a4e4-978e33432dba,"" +5253d689-c7bd-441c-ab16-3721f1e3b5cb,"" +527721f5-443d-4367-84e8-0f15982e7587,"" +529479a1-36f6-4133-bb4e-2f3cf677626d,"" +52c03be3-5715-4edd-9809-ebff7e2ba57c,"" +52d51fd3-d0c8-4046-9ec3-1a3f523d0b75,"" +53010db2-bba4-4751-9f12-f51429c9c558,"" +53339064-e7f3-47b0-a5c6-ce77d50b7489,"" +537bd7f9-9f4c-4bf1-bc21-9d09dd655a13,false +53e5edab-cd6e-42cd-b6e3-a36d57be3685,"" +5441b3d3-b8e8-4e36-84c5-623e9b199daf,"" +54487624-db9d-43cd-b4a6-0b4d3934f7bb,true +54627df6-151a-4f3c-9fea-374217afba2b,"" +5496c32e-97eb-4f6f-a7ee-46bd2123f966,"" +55054683-a9bc-4c84-b1b9-6a1540e1168f,"" +55449cc5-a88a-4181-9550-5d2bb23a293e,"" +5565df4e-b483-407b-8816-ca9e8ddefc27,"" +5578c18e-84ae-4f7a-8cb6-ce285e1b1ff4,false +55936779-3082-43b8-a1e8-a8329bf19a8c,"" +55954c84-4a6f-4667-b070-5fa9185fe3d2,"" +55dc6b95-e01c-4812-a812-2cbf20de74cf,false +562f66c7-19ed-4d05-a14d-6aaa37e6eb16,"" +56617f73-b5fd-4e8b-8bd5-467af440f0a9,"" +569f7616-50c5-47c5-94c8-fa3b3097bc97,false +5703a2ef-ccc3-4ca0-b100-a1c567f1dfc4,"" +572e67ff-4f0c-49c4-b55a-36a958d3e02f,"" +5742e203-dd0b-43c6-9dfa-304240238737,"" +574873f1-84ed-4171-a51b-6a2c609e9afa,"" +5760bdfc-cbd9-4763-841a-bb31a972f551,"" +5781b9ed-eeb3-45d3-a364-43cf8ea348f6,"" +579a0bf9-87da-4887-9700-a5098cd7d720,"" +579eb155-01ea-4066-9dae-9ddce5ea2041,"" +57d84d96-a1fd-44ec-8460-588b477502a5,"" +57fa04a3-a8e7-4699-9115-95957be21dea,"" +57fec813-6327-4290-a2e2-ca74a04cccf8,"" +588b882b-0bff-427d-8b34-294f2fe55189,"" +58aa2384-e80d-4083-a173-ba534f152680,"" +591075a3-92bb-42b7-a6b7-d761add1ad2f,"" +598288b2-fa85-4bf8-9e01-2d0e86711aa7,"" +59857d17-d8e0-4932-b92c-97bd36a31d3d,"" +59a25b17-fe55-4592-abfc-0c22e237bbd9,"" +59b131b5-2db4-4877-bcc1-2206865df718,"" +59e29c4a-102b-46a3-a950-3babee7cfccc,"" +5a0eb3ab-86ce-467f-b389-94f026fe0f7f,"" +5a4bc06f-8ed6-4cbb-93f7-5c5faee7e855,"" +5a9300fb-d831-4b67-88b2-843d2eca4441,"" +5b0954c2-a2cd-4013-b866-a6e0474c12c6,"" +5b2730a8-6a24-4bf7-aacd-f56fd380c696,"" +5b44153a-1ea5-45a1-9d58-bbc0a474ce1d,"" +5b5f6d2c-1899-474a-afae-a98762630b06,"" +5b72c7eb-4584-45c0-9039-5fc762b1e50d,"" +5ba15ba9-7112-45ab-b7e5-353d52af5fe3,"" +5bb09da7-8ce1-4642-a113-fdd1fbc73897,"" +5c085687-51d6-4cbe-9b71-0a58718c1d24,"" +5c17c3d7-a7ba-4756-9991-97bc0b3a46ad,"" +5c48ba78-4e71-441d-b03e-ef305d9e2bde,"" +5cce02c2-3537-4d34-b085-7cf145590292,"" +5cd558f0-20d1-40c7-8197-5a41ef97e634,"" +5cec04b3-e52d-4a6c-af98-6ee5d104b20a,"" +5cf73976-33ee-44ac-aade-5c64e56d7cd9,"" +5d54c744-365a-4db7-9675-79d64bdf55a6,"" +5d5dad82-9b58-4302-a494-02f44c3e94bd,"" +5d790c80-5b87-4802-9b89-02f949846f39,"" +5d883f31-f417-4d8b-bdbe-2cf73d1aa4b5,"" +5d9e8c9f-8c95-4ca2-9de2-2cf875868ff9,"" +5ecf975d-fa2b-4c6c-a5e6-fed730c9e035,"" +5ef7c113-d5bc-486c-9be9-aa39aea779d0,"" +5f6ad8e2-eed9-4b2d-8697-4e2e742ee1d4,"" +5f92feba-5842-4894-907b-9ceaf36d0f38,"" +5fc1d33e-168e-435c-93ae-c619999b6f9b,"" +5ffb50f0-0963-43d0-b032-3e6b2621e1d7,false +602f4e55-f69c-4103-bb6e-7d23fd1b06ac,"" +6042dc41-8cdc-4825-9ab2-bb22af5c5fa6,"" +60635e6e-3d74-4f56-acb3-be8edc522837,"" +60959eb0-ed9b-4043-b8a1-0b0f27789642,"" +60996e5d-082f-4d1e-8cc5-6cc626974925,"" +609a5704-81f0-4b67-881c-d00e87b74c0a,"" +6198d258-17e2-4956-8064-1448f4744303,"" +61e75b9d-46a2-4d74-a0f0-37dc66a09450,"" +6220b92e-946e-449a-a2bd-3241e6a453a9,false +62346d78-6be2-4cfb-850e-6ca1fc3f5e35,"" +62348772-d9cb-476a-a016-32f16e44cb33,"" +62ad1938-6235-48d8-aafb-2f773a40c6e1,"" +62ccafe7-fbcb-4cbf-8584-224cd58ef421,"" +62e4244f-3f34-48f4-b865-c8b1c3ad97f5,"" +62f342e2-5499-47c8-959c-4d786aa5fd67,"" +62f6ce69-77ca-436c-9d40-3eb541853798,"" +6320229b-25c2-4355-a288-17efb054df6f,"" +633dac04-7a54-476c-8482-9631c351ac26,"" +639dab1d-3282-4de9-84f7-727d0462f11f,"" +63b2b9b1-aaf2-4539-97f6-aa6815e75f21,"" +63d6c1e2-ce14-4e89-9bcc-f3e3e0b6fe72,"" +63e5b7be-525c-4a0b-852f-4cb090178d30,"" +63e709bb-a56a-452b-a48c-5125a367572a,"" +641068ee-0805-4403-a159-4fdb185cfdad,"" +64115066-e659-4cd0-9a56-fcd43a6df27d,"" +64148c8b-89e4-4394-b044-27088ab3adc6,"" +6433bbab-cad9-4e98-b488-c06e37acf268,"" +645572a4-5806-4854-b89c-939463a904f7,"" +64a2df8b-688c-4368-b672-0406ac7032ad,false +64c6ac23-d619-44aa-88fc-969376bebfcb,"" +657aed74-c373-4c45-9de7-b75cdf62577f,"" +65ad0ff6-ba27-4eea-a400-3d96240a5702,"" +65b9a660-2f5f-4d9b-9f15-5e572d65ee9a,"" +65e87fc6-9232-4ba9-baae-61d7af6e54f4,"" +66037242-eaa2-43af-98ed-190c3079833e,"" +667d62a5-160b-4657-be40-beeb787d51d7,"" +6683a92d-2d09-4875-9403-b977bc2eb37f,"" +66be2f9e-fffe-454e-86ae-75dc713ee934,"" +678223e6-745c-4d58-888c-b910dbb9039f,"" +67cf1dde-11a3-4aff-a7f2-a8b8288aac8e,"" +67ed7f4f-4873-409b-a04a-b928cb461719,"" +68059a10-b5eb-41c5-bb44-c56ecdb6baf1,"" +68249708-6a5c-474d-b1f1-214514fe5dbb,"" +682b9750-e94b-4522-b277-0d572b67e94d,"" +68935870-c60d-44c5-b57a-000c0a99e9d2,"" +68a1d32e-915c-417d-98c9-3154ffbc9249,"" +68a84b5d-edb4-41ee-82aa-8612ab7b2b38,"" +68dc301a-6b81-449d-bcf0-7b2f3d3335db,"" +692cc2c1-44b0-4c47-86b1-2d3c4a5848bb,"" +6938bd7d-06d6-4a9d-8234-9123aa238788,"" +697d10ec-d72e-49b1-b2f9-ac8a3be5c856,"" +698df1c2-7cc5-44cc-ab61-5420d92c598a,"" +699d6871-1043-4581-8dfd-fd1b6e47d518,"" +699f0f86-fc10-4f15-9a78-19515e4e4958,"" +69ae7f71-a967-4c7e-a046-cef77b622f70,"" +69c3f870-e1a5-4785-8dc4-067a7590b949,"" +69e71f61-bde7-494c-b65e-346f5972d21a,"" +69fe60c1-d372-41ec-8671-94090b970a7f,"" +6a4adab1-6b87-4e61-a400-dfa4aa4af613,"" +6a50955b-b259-464b-b3ab-9fa173caeaa5,false +6a8a69b4-f6d9-457e-8466-fdf87ac0585c,"" +6a967445-47ae-46aa-8754-2e48915efc2e,"" +6aba4aa4-9fe6-4759-9f76-705127be70c0,"" +6b8c1da8-2bca-4e16-a04c-a99a04c76d85,"" +6b9af3c5-cc2a-40af-9264-1c68ea98d031,false +6c18b368-8579-42e7-b0da-3500ed15ab82,"" +6c3be815-1494-4c82-b655-a3e7c92ee945,false +6c7689e6-db2a-4544-bf19-0b70c6d74279,false +6ca383d2-551e-4c89-b07f-c0faf9b36dcf,"" +6cac250a-d650-4c9d-a896-b38dd7e72955,"" +6cc22c4d-6014-44f6-a1e9-0b8eaf3f837a,"" +6d2776a9-6add-4222-bb43-64bd4dda65da,"" +6d34e7f6-d885-49f6-b072-6d06f6677a2d,"" +6d564224-4c2f-4bbc-af57-9895ad3b57d6,"" +6d5ff978-ef70-4eba-a1db-536985901dce,"" +6d759795-d565-43ce-90d5-91b167b18c10,"" +6d813c64-a15b-4e50-af82-c2f18f82a0cb,"" +6d94712f-7565-424b-ad4a-e1789fe2a639,"" +6d955124-bd23-4958-bad6-b36cdad50a5a,"" +6d986c2e-b12b-4a6f-ac80-d3bcdaede076,"" +6e02f522-f081-402b-88f2-38e6622bd006,"" +6e362fd8-b0b4-4cff-85c3-6586920627ad,"" +6e750faa-cf8a-4f5e-a02d-1471741cfdab,"" +6ea2fcf1-31dd-46e4-be06-592626e212f5,"" +6ea7ff7c-35f3-4350-9f9f-104c5182cd0b,"" +6ead6acc-52c8-4df1-b00e-b46e2e2f0a04,"" +6ec2fab5-4d76-4c80-ab2d-577b041c1aee,"" +6f550b68-a7cb-4314-ba74-d7d2a6d56d0f,"" +6f63c413-6f3c-48db-990b-fcc749cc6355,"" +6fb0d554-b14a-4bbb-aa5f-a4d5c082c2dd,"" +6fd78455-d026-412e-ab31-d9e1a02923dd,"" +6fdefa93-f23b-4eab-82d0-1e9ce7c6c17c,"" +70777428-a2a0-4dd6-b048-1ef2a27d9ec6,"" +7084e394-9ee8-4b0e-a3f5-e8d091ab4fae,"" +70a25de3-45dd-4848-a394-e87d4f6b6930,"" +70af4c33-277a-4a9b-8022-60d6bb0ac6aa,"" +7102c0da-2afa-4da2-a79b-a82742e4e3da,"" +713231d3-d739-455e-a913-41d5a1dc8571,"" +71f73b19-86e6-494b-8038-46a67396de4b,"" +7227c5bf-3699-48cf-a0c4-fb8f5d50540d,"" +72619c45-9cb0-4abf-a98e-20cc49a4e4f8,"" +7287ec50-e64d-418f-a83d-0d3d93981f62,"" +728f469e-7a39-4a77-8118-944f83c2cf21,"" +72972472-355f-4538-b69a-23aee8614438,"" +72baabec-7047-49ff-9e67-338361a06e5f,"" +72dd2d03-e239-4a2c-b509-769886057217,"" +73364971-4250-436d-b85e-93df377b2911,"" +734b53ff-038a-4c79-91e5-c14340b43145,"" +73fe21cd-4493-4afb-ba72-376f75bc2469,"" +7409225f-77a9-474b-889e-38eb4db0da2f,"" +74556136-3dd8-4074-8e26-160ae1b5f780,"" +748bb216-e420-46e3-9aef-6fe6b2020a74,"" +74cfd92e-5da2-4fae-be59-1546f5a22e35,"" +74dbc21d-e056-4050-8f0f-04194ac43995,"" +74eca765-7327-44ac-9868-ed01c1dc163d,"" +751225c7-3a03-40d5-980c-89f6020bc2ca,"" +752045ea-d278-4939-a82f-3ef29dd7616a,"" +752b713a-e049-403f-9d4a-f560ee0d4681,"" +752f97cd-9398-4830-ad13-38e26e3a8bcb,"" +753ae998-68fb-4917-93ed-484ef1753d9e,"" +75509cde-925c-4645-9a91-c1f244863842,"" +7569ce07-0cbd-4a30-bab7-d5f482a27883,"" +75969f3d-bddc-4e9c-8872-a33f108027a2,"" +75a6eba8-9d1c-4de1-b084-00a5aff10eab,"" +766ee5bf-c1a3-4837-8ee4-0c81e7252c91,"" +7690d499-c3d6-4092-89b7-81b3425164be,false +76a04620-5da7-4914-a5b7-5a8506221f16,true +76b1d775-0f28-4bdb-b6cc-a2441e8e17b9,"" +76cb0eb7-3679-4104-8121-9e01d36de541,"" +76d885d5-454a-4541-8ec8-61f7972f17f7,true +76f4f348-f65f-42da-a338-1e1f28df0ffe,"" +770f3f03-8a4b-4e32-8443-49b14d54ad3b,"" +7714ce28-a83c-48dd-a84c-ba3565432818,"" +7728c15e-82ee-48c2-90a4-afa79e048a52,"" +77293357-c730-4960-a8aa-c25ea0775c3f,"" +773da43c-fa5a-4a21-85c1-724b9d72bbbb,"" +776cf7db-0b09-48dc-b089-2d24410ed4d0,"" +77a645b1-5231-4486-8f16-0470ecc3e49f,"" +77a7196b-7391-4a38-89a1-e01b15e9e2a6,"" +7879670e-7953-4dc8-86ce-88f346cee653,"" +78a727b9-5bb5-4ed3-aec3-a3f804c79734,"" +78bfe231-0661-4502-b18d-28c24b223dce,"" +78edeaed-91f4-4f03-ab2d-38b720e5e6df,"" +7914b992-59e8-4541-8250-427de113802f,"" +794a6b0e-1e0f-4c12-ae52-47b53f385a03,"" +7957634e-ec90-46dc-9779-fb000649bd40,"" +795d85f9-c2d4-4219-8e06-f6fbcef30e33,"" +7968bcd8-e538-443f-b4a6-e41b9766965b,"" +797ac9d5-9ef4-49bd-af56-1bd0c971467e,"" +797d8c9f-56eb-4487-ae30-5c4dca379bf9,false +79813005-389e-4671-850d-b9a5fbc2345d,"" +798cfbc1-a7b2-47cb-91af-43af0ae0119f,"" +79a428bb-83a1-4338-a613-9b732f6feb4d,"" +79c5ee51-42c7-4fc0-8e8b-7ce9acb5133b,"" +7a281b03-7145-409f-9c5a-c45b6e544a4c,"" +7a543c90-f6bd-4343-8311-6818c3a9066e,"" +7aa8cd33-8365-4c53-ad1c-e07dcf2f86dc,"" +7ab27dbd-f952-44d9-bd9a-f67d5790fc82,"" +7b4255ec-4801-47ee-b12e-4b874c5eec1d,"" +7b8c6a88-3a53-48bd-9b73-bdc3f087e6b7,"" +7b9a52de-eb76-473a-a622-4b19d50b6bc9,"" +7ba6211d-542d-4508-96fc-712328eefc69,"" +7bb75949-ef13-4934-a19b-798f6a5e7866,"" +7bdf88dd-7924-4b93-9ff3-d543a170e0d8,"" +7bf5b1d8-3edd-458a-9730-187ac6738857,"" +7c197c69-7872-44cc-93bb-c518b8403da2,"" +7c6c40db-df29-4b8c-8881-3f1765e51c2d,"" +7c6e1d99-bc80-4172-b76c-226f6a618b48,"" +7c85bdf0-7156-4555-aa37-49e4195ce0e4,"" +7c94eac0-d189-4107-9443-452d124db20f,"" +7ce02db8-4ca5-42c9-afe5-b818e42e1767,"" +7d4a09ac-d7e0-4675-a9f0-313c73fa7906,"" +7d83ed97-4c6b-4ba0-9dd2-423f59e8bba2,"" +7da3d62d-56ee-4707-b036-2c1b5e111aff,"" +7de9fea9-9923-4dec-9c83-1cf13b81ca39,"" +7df8f003-7458-4083-8ef6-20a049526c19,"" +7e194149-713e-46fb-ab75-4a7cc2acd94f,"" +7e618ba1-1acc-4c2d-9b80-fee3d12884c9,"" +7eb4427c-1b25-405e-8ef1-38fabbfb1a0d,"" +7ebecbcf-c963-4cae-9b1a-8fe44e175008,"" +7ec8d0b4-41e3-40c7-8f18-fa10b249e023,"" +7f054bf1-9bff-4ade-8841-ebc67d4299c8,"" +7f35f61b-9551-45f5-9641-c9a74d7ef110,"" +7f8f71fd-602d-4ce0-ad32-211f5e7ca765,"" +805e5d13-6a31-405f-b43d-3449d73a50d3,"" +806085ba-3993-48ca-851b-a5dbe0690926,"" +80aca9f1-6907-4658-b0e4-bf6f53838148,"" +80e8b1f5-ea5b-4abb-9a06-b51bcce673e6,"" +81155341-1ad4-4473-ac14-9257704203e4,"" +8118e76c-1974-471c-9cf9-0b31ed231efd,"" +8131895a-c6f1-4b0c-aba6-cc3a52066646,"" +814970aa-0552-4357-b958-d572f56421ac,"" +8177f823-7335-4da6-aaf9-354e05299d71,false +8197ccde-83c1-4ab3-bf4b-e849667f1f31,"" +81dd2be8-9b50-4d63-bc4c-22c4dcba127e,"" +81f1e8f4-a335-4910-8a27-151141514c39,"" +822dee6c-337f-4a5a-a22b-7de9e64d1144,"" +822e3fd3-ae35-4e89-a5be-feac7c6f62cd,"" +8242d422-fa39-4374-85c7-5d267d68a0a7,"" +826cf99c-9d92-4cef-b717-d5c16ea7525c,"" +82e58dcf-ea72-4a50-9eab-a74460d58e4d,"" +82e8a61e-9eb4-4fce-ad43-881202cdfbef,"" +82f09a48-3b8f-4a2b-9067-61b51d85c500,"" +830df854-5f4b-474e-9aef-e7be6b452a92,"" +832e240d-da2b-4658-b638-cd2cf2f08912,"" +83648485-484e-46b6-9a11-64a48fe471f0,false +836f63ac-40e4-4170-a9f0-f170c4654db8,"" +83b971fa-bf7a-49bc-b1d9-09680e8eeb24,"" +84a15a42-428d-4092-9f54-235b55b11de3,"" +84c41fdc-0d34-4d95-8999-7677e4235e5b,"" +84c90e57-2348-4843-a557-444e3a82ce43,"" +84d36e49-fc69-46a1-a5be-e9842e8e192f,"" +84ed43cd-526c-4055-9422-c31dfefa43ef,"" +84f38578-0070-46e8-809a-65c75623829d,"" +8502fd55-eeba-4c91-8274-4f34dfe89977,"" +857895a5-3a82-45fa-9fc0-00380ef01b47,"" +85827568-1a34-42f7-877a-99a2d3a0f5c8,"" +8591ca33-6c89-4352-adf3-8261ce1bf58e,"" +85ea993c-f6d7-4b7f-836c-3c76206d66c1,"" +8644762e-70c7-45ba-a0fb-92fdb9e6c73b,"" +8668b20d-ed68-4d67-91fe-d35c385a3408,"" +866bd41c-debf-4a53-9108-fd9c0784103c,"" +86786682-7825-48f0-bbc3-3b1a127aa654,"" +86859d13-26a6-429a-a38e-be69f1af9b4d,"" +86946982-ff92-46cf-b6cf-caa45ba4a986,"" +86bc9f92-f89b-4fb1-affd-093a785ac1db,"" +86fee3a6-5135-4265-8c8c-04ac00ea8e12,"" +87061f96-823c-4e0e-9dde-8034682651a8,"" +87346977-9c76-4d76-b8a7-23ded71369a2,"" +8746d3f8-1943-4ab0-af49-f7d40aad5cc9,"" +875be020-8c24-450d-9ddf-f642e60e62f2,"" +87e5046c-332f-4ab2-b8a9-ab15cfa91fa5,"" +88274e29-0d4c-41c2-9315-f96c93d44c0d,"" +886a561a-e64b-44f8-bc77-76724e0c369d,"" +888dfba9-08c4-4048-bb9f-23fe80dfcf5f,"" +88a3a7ea-728e-4bbb-be44-73093e564cce,"" +88aecc11-264c-4a3e-a2b7-94c99d39fde7,"" +88cf49a9-ec8a-461c-87bb-003866a4e28e,"" +88db01b2-8bcd-4671-9447-f466279f21f8,"" +890cdafa-a6ba-47df-9cc6-5ea4d4febcfe,false +893f9661-bc71-4f65-83de-6483e73bbce3,"" +8979e8fc-9f34-4305-80fe-122fc619c7dd,"" +89cefdfd-b6d9-41fa-ab28-50563fe44a74,"" +89da332a-d11f-4f77-977f-6275f337495c,"" +89ee0500-71a0-47c2-a59a-1b681d9a92fb,"" +8a31ea5f-804f-4b8b-8f91-6ac4bea2c4dd,"" +8a56157c-ea64-4f80-81db-0c89a9d0b547,"" +8a5eacbe-4ee5-4e3f-9a0d-d0d1a13ae2ed,"" +8a8432ce-47e9-4fff-b71a-3147ad298fd0,"" +8ab69964-40a4-4d5e-8d2d-82d07174989c,"" +8ad34764-84db-4613-8483-4e8281f8f984,"" +8bbcf675-5176-4df5-bbeb-38603e8d9c42,"" +8bf02dec-bd39-4a1d-b074-e0ef2167b93a,"" +8bfb6668-fc94-4b2d-9ec2-3385f1780d58,"" +8c39345a-45b8-44db-b5f3-6b5914b3fd93,true +8c492ef6-59bb-4ccd-bc64-48d58d3da475,"" +8cda568d-3e48-4e77-afa1-374809ed3c3a,"" +8d671817-97a1-46c9-b23b-e3013f13edb4,"" +8d98567b-2930-47f8-ad39-759745179c38,"" +8dcdbdec-4dd9-4928-9e46-4f46fd6df9dc,"" +8e1303b5-0694-4bc3-a0f1-559d51e05364,"" +8e55e68c-f1ca-468b-b4f7-11c11c2956cd,"" +8e5ebd37-bb67-4c84-b9ec-3422ed7c3b19,"" +8e9c8ff0-82b8-4974-91a7-888dce4f7f8d,"" +8ea94f05-b5b6-4aeb-987a-567f627a4216,"" +8ef5e898-02ca-4617-b966-d5954b2cf403,"" +8f051932-7f43-4faa-b3b4-c2254756c603,"" +8f19178b-b9b8-4858-8f94-c0f1e6eaf2c9,"" +8f5f16b4-fcef-4a03-98c8-a6aecea249f0,"" +8f62568e-7778-4729-a454-369f5a3f12f9,"" +8f62ada0-3878-437b-aeca-d59d658ece1e,"" +8fc9a4e9-9ca1-4144-8cf3-032a5ed6bac2,"" +8fea3b61-246a-4333-ad04-78348cb59585,"" +8ff0c7a4-f1c3-400b-bb31-c66df4866cfc,"" +900405a8-53f2-4a6a-bb75-80e9725f3439,"" +90126aaa-1dde-4136-9be1-4e30e2c1b667,"" +90e6ec4e-bc73-413b-a07a-e8d3b675c027,false +91357869-20f4-4e58-8237-5b028e1f733e,"" +915eb7b9-47d0-4abd-a424-29605ef5f497,"" +91874305-9f9f-44b1-bac3-28dc0fabd0d2,"" +91f7217e-5c7c-4462-9f7a-ad7234bddf6d,"" +929edb45-2558-4fe7-b87b-ad4a801a26eb,"" +92e25da6-dacd-4d0c-9457-751bc135c126,"" +9307ddb3-12ef-414f-beb2-62a145d3f63f,"" +933302fd-1717-4cef-852f-0bc28dcfea35,"" +937de252-6da2-4dae-a80c-8ef39c6eb99b,"" +93851c17-f570-4055-b6bf-ff928624666d,"" +9389f1c7-bba0-4286-ab0e-73ca1430824b,"" +9391e021-6734-4c01-a6d4-a9e151e58201,"" +93c74a74-55b1-4f45-8b7f-3a0442ea5006,"" +93f749e0-9ee0-4cfc-b970-20f1757aa207,"" +94b95bc4-3355-4881-93ed-5db3dbcabc26,"" +94c1cc51-4dc2-4dd2-8a48-b6793675a44c,"" +95accbba-6ae9-4e70-9984-235e7371234c,"" +95ce9659-ea2b-43a9-a6cb-0b85fb4d9a48,false +95e68649-5482-4d3b-bc6d-ca77d83531a6,"" +95f21d98-e820-4054-970e-ab04c6b0ef69,"" +960866ac-7fbe-4f4c-8a41-f5c6d756a394,"" +962aaceb-7115-481d-8496-5ee5d12ac43c,"" +96526997-d779-49ea-999c-43bd07d250e0,true +96a5c8b4-5c9d-4092-9171-03da2422df80,"" +96df629e-35be-42c9-ad60-2fe43dfba0f1,"" +975a8040-5330-47c9-9bb6-13a1d07fd3c9,"" +979229e7-5022-4a8e-9f23-8673564bbf4e,"" +97bd3c96-d38e-4a46-9e9e-0ff3466b80a6,"" +97d8b079-ffd3-4bfe-a1a4-da06038813eb,"" +97f558d6-5624-426b-b21a-6a92c463d004,"" +98216728-4494-43ee-9af2-930201bcf2fc,"" +9873061d-c9e3-48f8-9a27-6a92255869c6,"" +98da112b-cca4-4bfb-9b85-32accfe1d295,"" +98e90113-a97a-40fb-b6e3-dd5f2e646901,"" +993d7bc1-e96f-477f-b7e4-ad8d25c59e94,"" +995d662c-8641-4089-bb52-417a2eaba272,"" +996a443e-1ca6-4e05-b1d8-b84455f2412c,true +99893b11-009f-4ae2-9ce4-c0db476800bb,"" +99929bed-454e-4131-8368-9d152ab1da8a,"" +99b31b91-ec68-4c94-b23f-2b75d9302cea,"" +9a1f1ac4-3ad5-4ada-80b2-cc31176393a6,"" +9ab5ad8c-b789-482b-838b-6cdcbd184330,"" +9acec191-7d26-4519-aac1-b50e00f3a2db,"" +9ad11a61-23e0-4603-90fe-c3001fdc7320,"" +9b4a1f69-52d8-4c2a-9e8c-072f71c2bd18,"" +9b876e7b-aa09-4b8f-8c58-a0cab74580b3,"" +9b9b76e9-88d4-4440-868a-c5b153c277d9,"" +9be9764e-cac0-489a-85e7-9f3c995e7061,"" +9c3ab93c-6424-4402-bd4f-0f0f7dd2b265,"" +9c630f43-c0db-42b7-b223-607d5a9a39a5,"" +9c9cebe1-ea55-48c0-9edf-22033f5ebebd,"" +9cb03b1e-f448-4f94-8c42-7f9271202a1c,"" +9cc11154-43fe-4bbc-a9ee-c5b8e0158d51,false +9cece101-e41b-4d77-be20-ae0c0cd2cb9f,"" +9d3c20e2-321f-488d-9472-5756fa061ca0,"" +9d415448-4c3d-40c3-b910-ee3ff0df9b3c,"" +9d495d7d-837a-4d3b-bb10-608f9abc919e,"" +9ddbf3cc-7d54-4532-a670-5c361df676d4,"" +9ddc0101-8632-4103-a12b-bf8c5590f624,"" +9deb306d-e6ef-4bdb-9a20-c9f5ebc556d5,"" +9e1b3c33-ef02-48ce-9b60-15b62a9a499e,"" +9e32e6c5-44d7-45b2-bb87-65e98f8464b0,"" +9e3fc42d-d08c-4cb4-93a0-a567c5f6a19f,"" +9e5071df-d3ea-4bd0-83b7-e08ba3fbb896,"" +9e58de28-a944-4ca2-b53b-b8da836ebd93,"" +9ea9ede3-8f0d-43c0-9f35-cc9fcfb622e3,false +9eac5165-a8e6-4acf-8d52-4ad49e2ba851,"" +9ed868fd-baf3-490d-b665-113b9427520d,"" +9f078010-f724-422a-8d8c-8855f2513eec,"" +9f3630e5-1b35-43bb-b53f-c191634ba14b,"" +9fa9a6d4-e6fe-4eba-bbe4-7319b735c63b,"" +a0238799-d0f0-49a9-96b4-33cfc1688279,"" +a0520e0d-7dc2-4218-9176-32eb842de85f,"" +a05480c6-8b4a-4406-9226-8dce4bbb4d21,"" +a0eff999-b331-46a2-b8dc-16727efc7580,"" +a0f58ee1-6e61-41a9-b0c7-5acdb005ac0d,"" +a1007ed7-d907-443d-813a-0bbb0ed84e50,"" +a1155987-73aa-4d48-b127-5da6d4cb7b02,"" +a1bd990a-7d22-47af-8258-5d4a05b269a6,"" +a1e0979c-d57c-4482-9087-88077cf657f9,"" +a2224a99-ae98-4018-8651-38f1f145c76b,"" +a2466ef3-db84-4434-9e07-af11c15c7490,"" +a253f260-5b6f-40a2-b4cc-d2c66856ffcb,"" +a27b5fe8-3465-4b6f-b63d-cca66626727f,"" +a28cbadf-918a-4831-be0e-9ab6c83133d7,"" +a2a6355a-2c94-4da3-a707-cc6db6553ce2,"" +a2bf4c9c-46d8-41c4-8eb3-044e832c78c6,"" +a2cb445a-10da-4b47-add1-16584c34d8e3,"" +a3240817-7453-496f-9d7b-058889858399,"" +a34042e1-cc97-476e-8564-382d941458b4,"" +a35cdf18-7bae-4776-8aaa-652bfc135af9,"" +a3b8ad3e-c982-4023-8977-e9717a33d645,true +a3f4de14-3398-4b34-9165-3a24c5742de1,"" +a3f9fe69-1527-4d04-a51d-ca9f1955d210,"" +a452f6e1-013f-4745-aa25-e2cd16ca4423,"" +a46ec80d-9cd6-49d6-863c-5f5425899445,"" +a49f38eb-8de7-49e1-b7de-97ab32fb277e,"" +a4b245ca-a50f-4bb2-ad64-1124c31eb166,"" +a4d85c0a-4687-4454-89be-085b9a2632f9,"" +a4fec49a-dc46-496d-86a7-5387c2490c3b,"" +a500b66b-fab5-4ee9-bda7-11fb6170fff0,"" +a533451e-9e8e-448e-bc5c-1beace2c397c,"" +a5a464c0-3b64-4ccc-bdb4-62aac198eddc,"" +a5c68c36-129c-482e-a9eb-023d7ec1ef0d,"" +a5d76237-0ea6-4016-91a1-33463ef1e9e4,"" +a60eddcd-ef19-4a84-ac8f-9206ed2aeed0,"" +a618792b-5b12-4545-a3e7-c07313d188f3,"" +a65e37e5-cf18-488e-8870-e4aeb021ee41,"" +a674a795-2553-478c-a5e4-6fbde13d7523,"" +a67aa655-62ae-4905-8042-30fd3ea68944,"" +a6a1fe2b-4322-4ea8-9ca9-3bcf288afe36,"" +a6d86605-7084-4477-8c8a-8cf25c4d1b9b,"" +a6fbb09b-5b17-49f7-a510-5c1596b424a3,"" +a717a4a4-b0b2-4aba-af9d-7b431e7883e2,"" +a71de636-d8aa-4455-ac06-932ae6d848ee,"" +a7575150-4148-4b9c-b941-64db8cc61356,true +a769b6ce-379a-4f6d-ad06-5df3a58d25ff,"" +a7a4ad54-167b-4cc8-8f88-dcb555c9dfb4,"" +a7b38940-3cd3-43b1-bba9-6adb6090d1d3,true +a82170ad-dc53-4899-a7ee-cdbb50e6a91e,"" +a83fed45-f814-46b9-9fdb-50892a1a1736,"" +a8624bea-ae5f-4fd8-b6ed-11b0fbba9254,"" +a863d462-6836-4f42-a4a7-8925574b76fd,"" +a8c9520b-bfa8-4915-bd66-a6133e1c5ffe,"" +a8f862f6-2e8b-4cec-81ab-eb3be51d71a8,false +a906f033-5f9b-4c1e-aaff-287cffc6cc7f,"" +a93d01a3-5bc6-40c2-9895-64a21fbd6253,"" +a9584e1e-deb8-45f9-830c-bef1caeb21c8,"" +a961ce1f-cf0e-42f7-ada6-f283daeac650,"" +a972425d-8c4b-48ab-ae2e-fc7db6661f67,"" +a9888a76-f2fd-4161-9d15-6126ba398899,"" +a9b103bc-f890-4d91-9714-8d674aa842fa,false +aa5fe96b-324a-4010-b128-a5063ce0be77,"" +aa86bd26-a862-4a03-9d8c-a04ec4c70c0e,"" +aaada565-8a59-4f69-b92a-a6cd9a323a24,"" +ab3e98bd-050f-4bfa-b6e9-d829291290a0,"" +ab50086c-5cac-4c6b-8e71-6988453228aa,"" +ab52bac3-d430-4700-b1e6-7efe2f5613da,"" +ab63c859-8b0c-4874-9404-cf7c89a368e8,"" +ab8aa7ee-f65c-4a29-9995-ef33c4135035,"" +abcfc7ff-d824-4414-a586-3c33c263a990,"" +ac139e4b-9d46-460e-92d6-0ac43ce8cb67,false +ac5bafbf-2bfe-4dc4-8e13-adb2d80e8dee,"" +ac645c40-0c51-4d77-a5fd-32e619019c60,"" +accb0b6f-b012-42cd-9374-91355ed49aa8,"" +acd3b2a0-1a81-4738-9bbc-9ab0c56b0a85,false +acd69c03-a3c4-44fa-b3d6-749476309db9,"" +ace8301d-b547-4cb2-8311-e27a8980633a,"" +ad07d661-6987-4928-93a0-e7b902b24ffd,"" +ad2a1d8e-6e19-4d5b-90a7-ab24050735a2,"" +ad48d6a7-890b-4eba-919c-ff6b63e35304,"" +adad08de-aeef-42cc-9a2d-b99fab5ab1dd,"" +adbb15f3-b71e-49a7-a954-2f117a93b972,"" +ade38b82-c9cd-4a9c-a7f6-23130baac87e,"" +ae13da32-7194-4f04-9fd5-c7f114a2e5b0,"" +ae4da701-069d-4ede-b59f-a2524c6529f0,"" +ae52de8a-deeb-4144-bfd3-52c2cab5fbb0,"" +ae56f2bc-f7f6-4bc3-ad8a-3b1193e40f71,"" +aea61272-bb6b-4fab-9817-584a8fa2f55b,"" +aec815b1-23cf-4504-8576-fe03b8798092,"" +aee86188-c252-4910-98cf-a8d3ca9d029a,"" +aef2f04d-7449-4088-9ab5-a227d1d7c36d,"" +aef45e8a-6923-4db5-9bb0-18f472f7f0ee,"" +af12db23-5b03-4e68-9127-acc14775690a,"" +af52f728-fcd9-464a-9e25-28013b84fa1f,"" +af73e4bf-2ba0-44ec-b619-a04f1c13c9a1,"" +af77032e-b1e1-42c5-8540-68bcb906f134,"" +afd2cfb1-57b0-44ab-a8f5-6edc92bc870b,"" +b0138fc8-736a-433d-831b-0bc39281d67d,"" +b0525467-772d-4d5f-b326-095fe72890e9,"" +b07bd4e7-6586-49d4-9f43-19d58889743f,"" +b0a1b45a-9dfb-482b-96d4-362cd1ee625a,"" +b0a510f8-2c98-436d-841a-eee086eba241,"" +b0ad5c48-9538-493c-b5a7-87f8f5ff858a,"" +b0b8b0c1-8184-401e-9346-37422a6f2f81,"" +b0d0e0b3-acc5-4887-a39e-df765111f99c,"" +b0e1d515-5be7-409a-8c86-3aab8ab6787a,"" +b101ebeb-ddbc-4bb2-b5c9-2bc6a23e0a16,"" +b109fc82-447e-4d4a-a447-1c7f0569bbc9,"" +b12f7f41-f8f8-4eb6-b2fa-dd5a18639c11,"" +b155b9d3-54b8-4906-a564-601bf123f027,"" +b18674a0-77ec-4d7a-b563-aad0856ce45e,false +b2071f60-d582-424d-b873-9ea4c2615442,"" +b217ba96-19d5-4184-a1ce-890bfdec478a,"" +b243e613-8498-427d-bc20-acc54786ed44,"" +b2503fbd-507b-44db-b4d0-32700a3ba5f5,"" +b260cc9d-644b-4e25-a99e-9afc44dc3879,"" +b2766451-0904-4ccb-90c3-fa7fcdbe19c4,"" +b2b2289a-0762-4846-a4a9-fbd7dc1a3822,"" +b2f8bfb5-d8a4-4df9-8724-bbce63e4dcdb,"" +b3394219-debb-4c75-988e-98fef99f0500,"" +b34ef85b-d2fd-4fbd-bb2f-6c21d737e21a,"" +b376a344-8079-4b95-99a9-d515f9dfa4a3,"" +b379efa9-29ce-44ad-b653-9cc03ac2bfe6,"" +b3861c55-ae00-4319-8a76-03627b8c8bf4,"" +b42a782d-79e6-4276-9fa8-744d5d1bea24,"" +b444d377-d1b0-4eb3-aca4-c24d3fac8c37,"" +b4771fd0-758c-413d-bbc2-17e8e2cc79bd,true +b48e7fa4-1922-4d4b-858f-a4f3bd8810a0,"" +b4a9bcfc-73ae-4798-a221-bbfe9991e0f0,"" +b4ce60ee-7591-48eb-81ec-0466d931b2a1,"" +b4f80b37-afaa-4974-9cd7-f53d788b2be0,"" +b508dc50-ca89-4b92-a12b-b032d3bf3a5f,"" +b516b7eb-4029-454b-a8ec-9bdede40ffb7,"" +b55a959a-73e0-4b7c-bb03-88bb22844c09,"" +b59c5c8e-7d5f-48bf-b3d8-900e18a64b12,"" +b67c717e-f9fc-4ec3-8cdc-ade1c8d1b95f,"" +b6d48a75-7d74-45e6-9fb0-154ca5264744,"" +b6ed4e6f-6911-4cd5-92de-f50e8d1916e4,"" +b7289502-ed78-469e-8b73-04811493a484,"" +b7597773-8d3f-4ee7-b54a-6c8b60b6d47a,"" +b762619c-c328-4367-99a9-41f10751cedf,"" +b7d87d24-1805-4a12-8f8d-2d14509d5eab,"" +b7f806c7-6ed8-4b7c-bf42-dae336e3f7b3,"" +b80e31af-6b74-4685-a83e-50c5d7f762fe,"" +b862117b-c899-4938-9df3-6baf43911d1b,"" +b89dd640-c21c-413c-8774-f9a154d897f8,"" +b9578448-2338-4f49-9d63-5b886a537f07,"" +b9701217-a4ec-4ef4-84f8-49ae5825b45f,"" +b97f2f71-17ed-4e3b-8e3c-374241e6faa6,"" +b982af8c-9e7b-4dba-ba74-76770efffefe,"" +b98be876-6fcc-41e8-ad4f-a78b1661acb1,"" +b9dc5e38-47af-4d07-8ad1-b0425c5cb386,"" +ba047f08-841c-4d5c-8f98-b3e614906d17,"" +ba5a0e2c-6e96-44d3-800a-991a81337994,"" +ba7bf8c1-e6a4-4437-b1ef-cf9e443971d0,"" +babdc581-4c93-4674-8743-1be4b18ba156,"" +bb182025-e149-489f-b69f-debf6907da09,"" +bb2c53fc-6d97-4747-a60b-8010f62b7f63,"" +bb770b6e-11d2-457f-858e-2e2c6c86c0bf,"" +bb94d817-e843-416a-bc31-4858001e7976,"" +bb9c95de-6073-4239-bb5d-023d18344157,"" +bbcd2900-31df-492c-9d43-e8e0243f7e1b,"" +bbe0b9ec-5a6a-44a9-9770-fbe4117593da,"" +bbf5580d-bef6-4cd7-a231-b6a532d68aed,"" +bc1bab6f-5d84-4c95-afa7-b4ab39038ced,"" +bc23a56d-f42d-4bdd-b704-07376d4a24af,"" +bc6abddc-4fa4-4ae8-97f4-9c566558b227,"" +bc75ec01-01c4-463d-b64b-597c106d4eac,"" +bc929bf3-06ac-408b-a17f-6ecdb28759a7,"" +bca14431-d108-4ceb-831b-b6cedb7b1b22,"" +bca1f5dd-414a-4570-9f0f-ed733bd170f6,"" +bcec2bc4-4e04-48cf-b52c-8e1fa2ac3875,"" +bcf5154c-180c-4366-836d-13aa6bfce980,"" +bd3b7896-2b3e-4c1a-a2b0-aca1c2ea23c1,"" +bd687828-857d-440d-836e-5230d3254b1c,"" +bddfe37f-fb3f-4a24-9ce9-181f250aa0b8,"" +bde29534-91ef-4817-a1f0-00a5cc9475bd,"" +be196f96-4e26-4c23-8241-b0a10a39e831,false +be1ca606-c535-4751-b52e-c61cfe9b6c73,"" +be3057cb-9635-4a25-863f-b4e79cda75c3,"" +be6d80a5-e2df-4e99-9721-80097129bd8c,"" +be7a093e-d70a-4a23-9db3-bd95aca334a6,"" +beee46de-ddd6-4fe4-80bc-6f1f4b8fd42a,"" +bef4846a-ca76-4000-a98a-00a9d327107d,"" +bf88c593-87b8-46d5-808e-8a332c5b50d5,"" +c015fb2e-4062-417d-86be-681d98f8a13b,"" +c0381889-4401-4bc4-9056-641340fa7ec9,"" +c070e6de-9ae3-42fc-9cf3-023839ad0442,"" +c086a87e-ec86-403c-8bc6-37a9b77924f0,"" +c095cfbf-cd84-4681-ba67-7b3ed318186b,"" +c0a2c5cd-58d4-49ca-baa7-d5859abc680b,"" +c1003a14-56be-43d0-9aed-41fb6b048273,"" +c1370f98-3833-49d3-b58a-21c581ac4ab2,"" +c14127c7-9943-471c-b5c8-ee9f0bb95bd7,"" +c1d430d5-32b8-4410-a1b5-6212b2c411ec,"" +c2253c94-aa7a-4e88-8410-e528b08654f2,"" +c24b807b-174b-4e5f-91d2-93599fb21548,"" +c25d3d58-1f6d-4004-85c4-3a162dbf48ec,"" +c286cd77-3c89-446c-a91a-4769aa088be1,"" +c297cce1-5d10-48da-8ef9-e860ac55f387,"" +c2a32eca-f03c-4fe3-8c29-3b69c9798b5a,"" +c34d92cb-7086-40e9-b303-97e204d2ab4a,"" +c36767f9-a3d8-4f57-a370-4a088ea0b023,"" +c37a2b82-a495-4d05-8f8b-99a757d0b83d,"" +c391f109-1259-47f6-979b-e72492709b66,"" +c3aaa4df-ead6-4a84-982d-3c28752e1070,"" +c3ab4714-5eda-433e-b461-c1710b769476,"" +c3de8ca9-c03a-4689-8701-dd524884b5b5,"" +c3e98495-15f0-4040-b92c-31aa40235dcf,"" +c47474bb-570e-45af-ad38-8f5352ee3774,"" +c4757dc4-d69a-4e2b-9b8d-61353a5b21e7,"" +c4e0ccb4-df43-4b20-873d-3a66faab42b1,"" +c4e31ea7-6ad0-42b7-8579-c3b7fcafa740,"" +c4e6e6bd-2135-47e8-81ae-7e1d6d5e7482,"" +c4f82d4f-f53a-4d32-bc78-67aa7786ed1e,"" +c52dbae3-07a4-4039-b04e-d79a86941c76,"" +c52de404-7bbd-4b07-95a0-a66df22a3221,"" +c533b9cd-e56b-4e32-93b2-1189f37eb646,"" +c557b460-3e1e-4ec4-9b4d-ea7bc0c8d64c,false +c5609af5-9f49-49f1-80ac-b18fccbc9720,"" +c5f20b88-cbd1-4283-ad46-5b0bf31fa3ba,"" +c6386600-4824-43b9-b366-4504f4e74d66,"" +c6442145-3aad-4b57-b2cb-b7eff0699959,"" +c6700fda-f009-496e-83ec-0532f34bfa92,"" +c6a0f008-4d97-4319-8099-7e0cb6708438,"" +c6a3be9a-b061-4a5f-879c-e948aa895f3a,"" +c6b44473-ebbc-48dc-8a42-44b7da0de840,"" +c6cb9c06-0c13-4cd7-8d20-ca7b99f7d141,"" +c761f0d9-c918-41b3-9131-856aad1f5b2b,"" +c7e313a6-4659-4fad-9402-946879a91616,"" +c7f263ed-22de-479e-909b-f6ceaf5f2eb0,"" +c80ab964-cf3d-4e89-9232-8e787cb0def5,"" +c80ec189-5f9e-4049-922e-969d28b3bdde,"" +c837974e-0093-438d-a822-19e97249f0dd,"" +c895019e-6471-423d-ba15-4f06ed628c7f,"" +c8bbf035-8a01-4c57-934e-988a584c98b9,"" +c8c936bd-3e6e-4500-bbd5-051659c59309,"" +c8cb1ae0-6a31-4869-a396-78dd5dc7f302,"" +c8f123b5-8e57-4c9a-8770-bd4b4c2c064a,"" +c8fd40e1-35fe-4c8b-af6b-730fd6fb5fab,"" +c92a4616-7215-4ba1-99df-40aea82eedc3,"" +c9445880-656a-4def-ad1b-2a4200d76c13,"" +c9647fa4-976a-4d9f-9be4-26b9c71f6eb1,"" +c98372ea-e9ae-4438-8715-b54ccdcc8f30,"" +c99c2510-b392-42ee-97e1-16f6a3b50978,"" +c9cd4daa-907f-490d-b3e5-a43eedd74c32,"" +ca45fe64-e7c9-48ea-bc02-3bc6a0b58e46,"" +cab21bd5-4314-4bb1-8b16-0967f8b08757,"" +cab53124-f25d-4b4a-a238-3d028e788b2e,"" +cac06963-2da1-4935-a6c3-d6152132c7eb,"" +cad24afe-7131-4158-9153-742c9fd16c9b,"" +cb1c6a1f-e659-4fe2-9cfd-f28e776a4f06,"" +cb5c302b-35a6-4d8d-8d6f-72ad1a3ed1cd,"" +cba58256-d558-48df-83d7-be5939d6dfab,"" +cbafa157-0905-418d-983e-827cc88faa94,"" +cbfd98df-8ac2-4497-988b-bc4d1c9c6095,false +cc0b8f72-76dd-4eaa-84c1-c343982e18e2,"" +cc1e584e-6752-4bf3-8c51-ac0192b5df44,"" +cc372f1d-a83f-438e-8d8e-c346eb03fee6,"" +cc38b529-3137-4d89-804e-51c6bd60ca97,"" +cc3a3e25-87b0-4012-869b-9905b31f8319,"" +cc5fd0ac-6ce5-43a6-bb4a-67279156ee8c,"" +cc7e521d-4fa6-4215-942a-48f84ddf9a49,"" +cc8ade38-d9f1-4700-aad3-dc866f35d1a1,"" +cca1c9e8-c68f-4fe0-ae79-693399d8ee31,"" +ccb56a74-40db-4d58-b0c4-cd485e8a7fe8,"" +ccb810a0-28a5-4248-92dd-7259a0fa0c22,"" +ccbda4c9-3ec5-46c1-a8d6-a215651dfb43,"" +ccbda933-c3d4-44bc-acfc-fd26bc234bb6,"" +ccd5afad-8f2f-4305-b646-26137eff0a0d,"" +ccd69a01-cd72-41e7-87b9-de8454411ee3,"" +cce63ba1-abd2-45d1-9ba0-a2cd0fa9b00e,"" +cce6dbc6-3a49-4f8c-b908-e6f11be81ac6,"" +ccf36cc5-5009-4fbc-8e8e-81f38d3195a7,"" +cd013d11-8609-41d9-a9c6-7fb33a499f51,"" +cd09f5d3-c846-41e7-a1ff-aabb55973663,"" +cd2c3f78-ddb1-45f7-9939-987e070d9414,"" +cd3d18de-96d3-4132-ae73-b80795a53ad0,"" +cd71ffde-9d7e-4c27-bc22-306f81cb46b7,"" +cdc96f77-a589-4ae4-85b6-c6d2c89487ba,"" +cdcad0ef-8049-41c2-a8ee-21835323b05d,"" +cdcae470-3531-4548-a2a9-ba665fca8930,"" +cdfbf7e0-ad10-4c22-9ee4-888bb6bdd198,"" +ce1b7543-fee6-4267-a00d-daa5a5ea2861,"" +ce225925-36ec-4c1c-a327-c64a146bef46,"" +ce512c4f-3808-4ef6-9853-6f7d14e146f9,"" +ce7037fc-2614-46f4-a8db-40a2a0f9e3ac,"" +cea2e600-da22-42b8-aa08-f0e951ff35d5,"" +ceea3ce9-5ab6-4cbe-99b2-34ca9e6eb468,"" +cf17f19b-9f66-4391-bd2c-55384690d9bf,"" +cfced5ea-1e8e-4c85-8318-ef91a041f4e0,"" +cfff2bee-7cf4-459f-8690-3638266a07e3,"" +d000b381-5ba8-4a5b-84c7-f371e104eeaa,"" +d012051d-65bd-4a59-b4cd-f454d6010888,"" +d05fad91-9a3d-4d8b-ba99-5a33fcc82081,"" +d083a1aa-9662-482a-8192-4709f904f16f,"" +d0b22c1a-d0f6-4d83-bd3c-475a3acb0b83,"" +d0be6a2c-ebc5-405a-a2a5-006a5ea52c76,"" +d15e2267-a699-4f43-96a3-e98de7ace044,"" +d168f31c-483c-4340-a2f4-6a60cc8cab30,"" +d17dfd97-60aa-4313-bcd7-f48935faa2e8,"" +d19385b9-bc04-4000-bddf-1d09425a2b4f,"" +d1ba146a-9503-472e-b668-3686ff275ef2,false +d1c2f4a3-8af5-4ad4-bd0c-032b947d3c25,"" +d21af0cb-c5a1-401d-b605-a2e3ba3f46a1,"" +d2272949-75dd-4949-88fc-f0f157c83105,"" +d230ce54-d5fe-4983-9e91-519ce1249322,"" +d241d156-478f-47a0-989a-96a5fff36780,"" +d2663716-cdce-4d88-90f8-51fb5d59aa4d,"" +d2a3d703-44e0-4dee-a8bf-9fc0457841ae,"" +d2c11b5d-93c6-4500-bc4c-8dc50057f362,"" +d2c4f328-e7a4-4d87-b180-9a0eee13d165,"" +d2d08017-f955-4e0e-baf0-2d5d37acc304,"" +d31bee09-f02b-44c5-aa15-a06343e0bf9f,"" +d355c55f-53b5-407e-8785-33e4e54bb3fe,"" +d37a07ee-5d7b-41d0-bbf1-331277662c52,"" +d397d6be-6008-47bb-9f72-628fb5c41cb1,"" +d3997124-9e16-4df3-8c60-f43f62bbf036,"" +d3e6de0a-bfef-438a-b982-4818145651b9,"" +d3f2c0c5-1f4c-4156-8cc0-e2d87db500ea,"" +d44741c1-d58a-495e-9f30-cc0f19fafc4f,"" +d46e0f9c-5dd1-4e88-9a22-96ca436674d3,"" +d4aa494b-3328-48fa-9873-39be6536a437,"" +d4dc8d84-20e2-4053-b244-47549e74680c,"" +d524b58b-90df-4d29-b4c8-308ac04dda2b,"" +d56569d3-0a16-4968-9470-93d410916f11,"" +d5944a91-3bf9-4915-8512-0f1f50a8eb7c,"" +d5aa4b50-1ac3-48bc-90ee-7ad6bd78a078,"" +d5ca4448-871c-4175-801f-38bb43cdb48d,"" +d60c26c8-1b83-4de9-be17-81fca1ef2575,"" +d6c622c5-3662-4b29-9180-0b4055afa612,true +d6f1073c-bf1c-4232-8ee2-5b92f0a32d8e,"" +d702593f-4121-4ac3-bc3f-dae097c7a044,"" +d702f666-bb61-4d55-9579-e56efc3e0e91,"" +d704d649-9e08-4502-a9c5-820f503dac9a,"" +d7c0876e-3031-4be6-aa2b-bbd8e2bd8884,"" +d7eba5f5-103c-4442-9a7f-62c81a9dd9e5,"" +d7f76663-5195-4de4-bc43-ee61e467fee4,"" +d8534558-7fea-4dcb-a1fb-2981b725dd61,"" +d86d32ef-f50e-46d6-b434-fd03bc807cc4,"" +d8a42c84-6930-4b12-8c1f-9a492d290454,"" +d8cdd01f-c8b2-46fc-8f69-35da565e1b99,"" +d909c299-c622-4c39-829f-415621e09eb1,"" +d94160a6-7943-496d-8b66-172e9ec6804e,"" +d97e5072-19d7-49bb-aaf1-f3a56816f3e3,"" +d9e59fa9-630b-4d97-aa71-4e03bd6fe1ad,"" +d9ebe2d3-2a6e-4079-8f59-43a327e68ebf,"" +d9fb7469-d50c-4492-8c1c-8bac1ccf81bd,"" +da33cd96-8f15-47f1-aeaa-846234a6b9cd,"" +da412e67-cea6-4ad9-8c66-75bf7bd9d0f9,"" +da438390-6d67-444a-be2e-354a21d30559,"" +da4c7a8d-c75b-4050-bbac-5edb50e48d4e,"" +da5de060-6e29-4379-9119-175b394f9bac,"" +da98f3f0-236f-4c41-b123-09f1fdb5bfc6,"" +daa0c0a8-91bc-41c5-9627-adad04a7b15b,"" +dabbe6e8-3692-406f-8a2d-f81217eb6964,"" +dac01aa1-44bc-4ff5-b123-4868f8f278ef,"" +dac87128-5796-449e-85ea-2c1f8e09ba0b,"" +dad4c0b8-0e02-470c-b7be-6477008d925f,"" +dae5b7a2-ad95-411c-9721-067d3c578262,"" +dae62ac4-666c-48f7-8587-ff9330a9e9e6,false +dafa1e16-2149-4e76-a693-88d2e7e29a0a,"" +db0a2a90-dc62-4e6e-9a46-88b9a3145ab8,"" +db16da7a-538a-4270-bd0e-1b4398a922a6,"" +db1788c0-f515-419f-ada6-834a37168001,"" +db17c5e1-6d1e-4b0b-a345-8dca770e7097,false +db5a6739-388b-453c-99d0-58cfce24d5d5,"" +db8a9775-faff-49c4-8b35-74e92245f439,"" +dba78a31-21c5-4600-b18e-3e90237ab40b,"" +dbbc9b03-4d76-4118-8637-185659ce2adb,"" +dbc0cb40-5e33-468e-bc9e-2cf8ed602d01,"" +dc2a8c49-ac89-487f-a22c-dc126a5c7ed1,"" +dc36732e-8054-4d6b-a860-e7c54f04a0bf,"" +dc567aee-5073-493e-bec1-b6b0f4e160d5,"" +dca15405-893d-4468-84ad-574e54f45c22,"" +dca8db58-f563-4a61-b747-94f5ec5bf135,"" +dd09736a-85a6-4c2c-979b-f2a7027308d1,"" +dd9a2411-d45c-4385-a6cc-650df658ae9e,"" +ddccfc35-d71d-4e65-afd3-4cf2a21e607f,"" +ddda9bf7-d645-40a2-8aed-eb9efa425d01,"" +de1c574a-a49c-41d2-b321-849c005c89e0,"" +de81cdd8-a6ed-4ed7-af7b-6ee027d0d5cf,"" +de9bffc9-c6ee-4ad2-8055-1e0d0e48887d,"" +dea4f32f-0f51-4bf0-af67-e79b34674fac,false +dee72459-d0ba-4771-af17-f38f8396adb0,"" +deee1173-4f31-4784-8c63-d02e2a7b16ba,"" +df471b44-cfeb-43fe-9488-8f85a03952fa,true +df529c86-f229-404c-8f0c-44e1b29e4e95,"" +df54eb2c-59c6-46cf-9cc0-201e94acc07d,"" +df6522a8-552f-4ba1-906c-139f58a0a85e,"" +df8a0f98-893b-400b-9da4-2514e50c0b24,"" +dfa10a30-82d4-433b-9df5-b0d2291378ae,"" +dfa72495-2737-4fe8-9fcb-c799ab51ddcb,"" +dfb67b9c-64b1-422f-887e-139a8f676fa4,"" +dfb6cb62-17e3-416b-9214-6ae6f432874a,"" +dfef1286-acc9-44ee-a736-b9676812235b,"" +dffc6fbb-8592-4f0a-8190-1e64eb2bc634,"" +e0010996-fa96-46be-9431-ebbe139c081c,"" +e00ec480-db7a-4aab-84ab-7e5e7184bfdc,"" +e0ef5017-54ab-469b-9871-f706d4899465,"" +e0f2d024-4210-4f18-82ce-51755627e13f,"" +e1309aae-d59f-44f0-b630-fc538bab9027,"" +e134ed0a-49e0-4a0d-b09b-4e874f422e60,"" +e16400e0-a6e6-459a-a16d-361a7b2ffd1d,"" +e1a9f42d-103d-4e01-b98a-e70f98aa1c00,"" +e1c6ba82-6bc7-4f15-b71c-c669cb78bcc9,"" +e1e4e2e8-1c7b-4c67-bc7a-3940e6670357,"" +e1f9bb17-3719-45d6-b4bd-3be88f17e429,"" +e1faaf81-1761-4a4f-b01d-c179db8474f3,"" +e20067e7-f2fb-472b-811a-3d28a08bf31d,"" +e2501541-dbeb-4e28-b4d8-598e963d6158,"" +e2b04908-769e-4916-bbc4-45ad7db85e02,"" +e2eccb3c-0755-49c8-8e82-cc510481ed04,"" +e2fbc85e-dbc4-422e-9a11-0007d5c194ae,"" +e3784666-7f48-455f-a137-e6262a617ccf,"" +e3c2c69c-bbe5-4678-aa6c-3e5c7ff996f2,"" +e3c5df86-3d35-4dd8-a081-48632a3b67c8,"" +e3c94ca7-a487-468d-83f2-8068a592a774,"" +e3e4b2a1-6c40-4f42-a5b5-17b352209b40,"" +e47d814f-5139-4339-a710-fe953efcfa74,"" +e484c47e-4da7-425b-a636-de512fcbc822,"" +e50c31e2-5d52-4d45-9b8f-8ae3fdd26488,"" +e53c4c4c-976f-4cb9-95af-fb11c264cb72,"" +e53ec08c-9029-4b50-bc1b-a302878b17e6,"" +e562454d-7fbe-43a2-8046-bfc5b6ab043b,"" +e5803192-2273-43bf-8acc-945cc13af70f,"" +e5c7eb1c-238e-4bf8-82bb-205c383f1517,"" +e5d9ff02-57a4-4e05-b573-0dbc80e19958,"" +e5fa00b8-1779-467d-b881-757b6796d8b1,"" +e5ffe973-b166-4240-b5d5-17eaad746c4d,"" +e6e9da2c-7684-41fb-9123-def8a7cb4dba,"" +e7098f3c-9dec-4aa9-9554-b3ad9a7d842a,"" +e73f4c39-965c-47ca-adb5-0544e20d45bc,"" +e757ee7c-a50b-4575-b383-587e6f782bed,"" +e780a3b4-6b8a-40a1-a9ba-5f95725399ea,"" +e78c789e-7b81-49ea-b7e7-d17d8e8925aa,"" +e79745a2-654a-46d6-a886-dd1d21acccf4,"" +e7b3ee1d-2d5e-4dd6-8ae0-ebddf36aee4d,"" +e7d53c39-4adb-441d-a4e1-50c5f60fd3ef,"" +e81ec213-328e-446b-9c33-ee80bc850d44,"" +e83d90d6-c694-47fc-8120-e91546fdda8c,"" +e8ba3683-aa88-4800-a248-737d39ecb6c6,"" +e8f05409-642b-419f-8475-28449c5f9569,"" +e952a654-0605-409f-ac45-19d00d4b53c5,"" +e97e9781-3cb3-44b0-89a9-b704d456fcc2,"" +e9ab227b-2a64-4c48-a8a0-6c166bf4b970,"" +ea2f0657-1625-414b-9c7c-bd0b01a12eff,"" +ea91b2ad-37c8-4a4c-9178-11d7a5e4e75f,"" +eac7851d-1d61-4a35-9f08-c5da53335f7f,false +eace08fa-4018-40fa-a7fc-3daf10b2f21d,"" +eae386dd-ef2b-4629-817f-6f8c44ce5817,"" +eb2419c3-3b91-4d73-b4be-a40e759ab957,"" +eb2781af-72a5-4760-a9a7-20d212f56230,"" +eb511148-31f2-4292-a83f-d16d3be6631e,"" +eba22453-8605-43d3-81f4-f703e58a230d,"" +ebad9165-a532-4846-aff7-5038cd5c694f,"" +ec3d7e48-c9f4-4f64-ada9-7e2891a62d1b,"" +ec462f5c-2aaf-45fa-8c25-6f556349af00,"" +ec4d4a14-0934-4d28-ad20-64db8fe78d42,"" +ec9ccc2d-1e6d-4ebc-b1e5-86d493d19eb4,"" +ed352845-d1af-40f9-afca-53503a6518d8,"" +ed7a65e1-de6f-4c2f-bd47-aa78d0c2189c,"" +edd41c4a-a5f3-4531-b506-d81c04e1ebc6,"" +ee004b78-c05d-4c34-a9f3-290a9259d3a3,false +ee6749cf-0622-4376-958c-229ebc0f9618,"" +ee8905ec-84b9-4fa0-b1b5-0908206b7af9,"" +eeaa2e25-739d-443e-ae83-b0ef04e2836f,false +eed7cb9f-7fe4-4f9d-bb4d-cdaeea6e144f,"" +eeeed70d-3cc8-4b11-9f6e-ab44da414b90,"" +ef079a39-9f59-4681-91fa-3d745c5eaec1,"" +ef10cf22-bdb9-44b5-bc9e-bb777ad8f090,"" +ef1c5dab-b3a9-444b-b825-8fe3d0248e02,"" +ef9360c1-b9b4-4dd9-86de-f996cd22e7a1,"" +efbe6910-0629-46d6-bcfb-5880a1795742,"" +eff17621-813f-4e99-a307-118ac9273efc,"" +f018a615-d5ff-4c3e-84f8-d4ebb1f342bc,true +f04f4b77-a9f2-4ce1-9d0d-6b43d39483f7,"" +f07f7d83-1fe3-4da4-9c33-86aae3ac67e5,"" +f08704a2-c1f5-44d5-a3f6-43c5608dc987,"" +f0a81ab6-10ea-4859-a064-b3b0dc662f24,"" +f177f59d-ecf0-41a6-b81b-a311eb760976,"" +f1a1aa8c-3779-4c42-8302-6006dcb95151,true +f1b36b35-1bd3-4328-aea9-4c5b3a7c7404,"" +f226c31e-37f4-4a4a-aa1f-aa400447a799,"" +f22fe03d-e1e9-40d5-9ff7-a5bb6e7a2d71,"" +f27187d5-f8f0-4188-ae0c-f77ca494a9b3,"" +f27a59e9-c785-47d7-88a4-de7706bb4f92,"" +f28962fb-0214-4664-b9c3-7c67d4eb2e8d,"" +f29f4e3c-81f9-49b1-bc52-47a60a3076bb,"" +f2fd6221-529b-42ca-a060-30db0996a398,"" +f3047ad3-3524-4a48-a00f-603a65fa0891,"" +f310aaca-1c10-41fc-ac49-8430c4c23656,"" +f314d140-929d-4ced-bbd9-c7f0204e99eb,"" +f32094e2-ab74-4567-98c7-e35fe529393a,"" +f3338731-d7b9-4c1f-8e3d-e73b5b756c7b,"" +f3deb81d-79ca-4176-9b6d-bd96bb5b7a16,"" +f3e99f4d-5bbb-4f11-9519-4d3aa15e25cb,"" +f3fd705f-3f03-4f26-a477-f36e65fe7499,false +f4077b1d-84fd-4489-b8b1-bbc9f5b61da2,"" +f475b875-3df0-49fc-b4f9-10b2ac1c40a6,"" +f4bba159-e786-4f15-8100-6bff75c03635,"" +f4e59f5d-12a4-4005-bdae-e157e145c119,"" +f5042323-b136-4c9d-bf57-0b7b70609495,"" +f51f7f2d-968a-4630-8e06-d6beb52b9f6b,"" +f55f863e-ffbf-4eb5-8f76-e2069098adcf,"" +f568f305-55f4-4d1f-849f-6c8f371f5051,"" +f5dec406-3636-4ac4-894f-1035ef3ef570,"" +f5e51d8b-bda1-4093-b887-f2355ea171cd,"" +f5ebecec-331b-4d57-b353-3f50538cae2a,"" +f5ed5cb8-adc0-424c-8371-ffb8e7cc9bc4,"" +f6514087-db2a-40d8-897a-21b5af7d590e,"" +f65d9eb9-e3d4-42fd-8d3b-168a6009f6ed,"" +f66efc52-e910-4d73-a9a7-fddc110c16c0,"" +f67e345d-7f27-48bf-afe4-00d79c3cf300,"" +f67f612c-6534-40d1-8e56-c056e886b98d,"" +f6b9b783-c8af-44c2-8443-44b80e972686,"" +f6bc9618-f245-4402-b69e-74d424736c4b,"" +f703fec9-ca54-4363-9ef0-4c4acdc37e0c,"" +f73ca57b-a00f-4217-9106-aedd26812b3d,"" +f74957d5-eef8-4683-976f-c59ffabdeadc,"" +f7d2872f-bfc1-496e-ae92-a9d446aa72da,"" +f7d6ad2c-1f7e-47ef-adfc-16a5d4d0189a,"" +f8043ec1-8313-45f9-a6dc-4cc1e5ad5f54,"" +f8673366-8964-4c6c-aa9f-02e0920f21dc,"" +f8683b38-42d3-4f32-a543-034c05dc07f6,"" +f8816dca-a41e-4ef9-8a17-b40fc7d707ad,"" +f89b5e47-c8c3-4067-99ea-bd22f8932ee1,"" +f8b6ebd6-8d2a-4592-ad16-06772ad32cc7,"" +f8ddb5b8-70ff-4d5b-adf0-1752811f7be2,"" +f8f99d6a-94dc-4fca-8772-d0c3fe887a7f,"" +f912a4cb-2eeb-41f3-a410-0ddc3eab9cdd,"" +f9185eb8-d2e5-47aa-916b-a5ff24612ddf,"" +f92f93d5-3037-4692-a062-ee84b9961510,"" +f93aa334-4532-4774-b82c-6d040d3c7156,"" +f93e975b-d07d-434d-b227-666e744892c7,"" +f95dafda-7f63-4ddb-acbf-e363dc901b1b,"" +f95f8351-818a-4cb1-b26a-507c4baced47,"" +f9aff2d6-fe39-4d17-90cf-917edf5d2bc8,"" +f9f5480b-27da-44fd-84a6-1ee4d4a55e36,"" +fa0ee43e-abc9-44cb-9248-3acc6c40e693,"" +fa4daa52-528e-45f5-8dd6-fe2a2c9fe8ff,"" +fa522b1a-3244-4203-966f-546dcfb0c62c,"" +fa5e8aa4-84e3-44b7-9dfe-326abdf02162,"" +fa7087f0-0140-4c94-b7c9-185b89c613c8,"" +fa786b6e-dbf4-4c51-a8c6-3006a029e8ba,"" +fa80db64-ef07-4b14-aa7d-e93585fd84e2,"" +faa80ca5-9c2a-40f4-aeec-26d982e663ae,"" +faeaaf02-2ae6-4f5b-b729-61fd56d3946c,false +faf774dd-13ef-4b09-8cc9-3cf33a562482,"" +fb176a2c-4b8f-40d7-bb81-f370c96c3f96,"" +fb1df190-d719-42b7-8e9c-d7830f92882f,"" +fb42fd3a-123d-4b2b-8a2f-d7e37af0a675,"" +fb55aea6-a5e7-4a55-a8df-967f504023f6,"" +fb93624f-3d94-4c5d-9aba-10a3eaea0434,"" +fba16e5e-1b98-484e-8296-43a3995b9f75,"" +fbacdef8-f2e5-4a1c-8a01-b25442a5af89,"" +fc348304-8ef3-4b3d-afcc-a66718315d89,"" +fc6dd798-66ac-45e2-a663-ea69b976c120,"" +fcdd2cbf-917f-4429-a41b-48f8fe2a188b,"" +fd325c37-4113-45e9-9e55-abd6b2ce6dbf,true +fd694334-1bcc-40f0-863a-017311fbaecb,"" +fd75e2b2-c780-477c-a813-046ccec6fe51,"" +fd7a8bc2-9005-4230-a4f5-348f4b1f5b16,"" +fd7ef037-c576-498d-bdef-cec86fdd60b0,"" +fdaeb95c-9d66-4da9-87cf-eabbdc593007,"" +fdb161bd-ad0e-47fa-a5bf-f17ece98f28c,"" +fdb49a29-6648-4dc5-bf56-ef948b00d885,"" +fdbbdafe-ed7e-4bd1-ae67-86430153a738,"" +fdc10bfc-2b5b-4a23-a93a-eaa85803bbc3,true +fdce38d2-ae3a-474f-8024-52279c43b745,"" +fdfcdb50-716b-4452-a4c8-e03d3b8f89a3,"" +fe4c4989-ff06-446e-bf01-5c5d8f69f1eb,"" +fe66bca4-eedf-42a9-8def-d29ef3dd01de,"" +fe719bdd-cc9a-4c93-b2c9-982d051fceb1,"" +fe7906dc-a882-488e-b576-2ef3486bcf79,"" +fe7c2db1-c229-4681-9373-e388dccb9e37,false +fe7cb896-d81d-4b38-885c-4134ea64020e,"" +fe97c34d-b787-4c3a-a972-821eb42a3442,"" +fead13f8-4b89-4f33-950b-ee9254a36c4b,"" +fead2d6e-2979-4de7-8118-698906954d06,"" +ff9df3bf-b8fe-4789-aa34-b205b5143a92,"" +ffab80f3-741c-4c67-af21-4585f1c05d2f,"" +ffb89ee2-c3d9-4fd5-b897-a0e0eb692143,false +fff8bb82-5151-41f9-b150-801ed4957277,"" diff --git a/fhir-server/src/test/resources/responses/ParserTest/lengthObservationSubtraction.csv b/fhir-server/src/test/resources/responses/ParserTest/lengthObservationSubtraction.csv new file mode 100644 index 0000000000..be423195dd --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/lengthObservationSubtraction.csv @@ -0,0 +1,1502 @@ +0020b035-7314-4867-9453-7921f158d003,"" +002593d4-d271-46a9-a900-1c498c322988,"" +00e9f82f-f16c-422f-81a3-77fa98958da3,true +00f02590-e737-42c4-9c73-206cf66b02b7,"" +011f5b11-f944-46ae-9cde-0d802e5c0073,"" +0143f01c-3692-479a-9a72-592260dbbd64,"" +01a4de83-37d0-4612-82eb-2ff2dbc2e02b,"" +01c177be-05bb-4a44-a199-1df387e68d8e,true +01c17bf0-9240-4149-983e-f0b3ccfb52be,"" +01d57966-1281-42ab-9e05-02d96d146b6e,true +01e27cc9-c9c0-4c33-82f6-9507394f5f2a,"" +0218f780-899c-4c1b-9167-45e33d02acfb,"" +021bf465-c299-4302-99ca-e29a6d25bda5,true +022ebe16-f618-4acb-9d99-88e23d686c34,"" +023a9220-3016-4202-8c7d-360d7d3bece9,true +025aa37b-4ae6-473b-9ae6-c12eece968c5,true +02741df6-79f0-477c-963c-8b3f8cb4374f,true +02b87530-10e2-4009-8734-4196642cef1d,"" +02c07924-f5c7-4909-ab49-dec34c051dfc,"" +02c4fa06-3281-46a0-9bb9-c7b99497c91d,"" +03365fa5-36d2-41b3-b3d9-1fea9f94c73e,true +03555f06-db5f-4b8c-aae3-f6d2c547f499,"" +03610d59-94a2-41de-a039-c671241b075a,true +038ced3d-5233-44c6-9a8c-1f6f9cc4be11,"" +03c7ba3d-8af2-44d1-a3c0-2c56b9fbc7c6,true +03dadf3e-cdc9-4f63-8a4d-89bae77bd529,"" +03e44215-f743-4c4e-8d19-aaf761f54503,"" +0429fff2-2839-45d8-b0bd-1d86aa92f966,"" +042bb1e3-e5aa-471e-9e83-bb4a1b90af22,"" +043108b6-98cc-4feb-8ad0-1394cd10a289,true +04367369-256f-4d47-b1ef-8014109ab46f,"" +04661736-24d8-41f7-9f05-af5682696df0,true +05048b00-24df-4311-806b-2f9a7cc16380,"" +0531e6b8-2ede-4e12-a91b-cfe7e12e552e,"" +054334cd-cfc9-4a1c-acf3-d0824cafd3a5,true +055824d7-b398-40b2-8fed-bc40552fffce,true +05643ef2-65e0-4812-b739-4fadfa3c066e,true +05bed469-760c-439b-9729-0479a6550b76,"" +05c37921-dde8-4b70-869e-82680975ab77,"" +05daccae-2d73-459d-8aea-7ee75966e52b,"" +0624683a-8572-4885-b51d-2197dde395bf,"" +062ab32e-318d-4ff7-81ea-f3be53204988,"" +068d13a5-8b46-4a3b-aca0-ac9ffe70f998,true +071b684c-70a5-4c93-8289-9630d25496d2,"" +072312b8-c984-4cb4-a4f8-97560368576d,"" +07279076-3064-44aa-b07b-346a7edf6abb,"" +072bd136-7e5d-49a2-b878-aaa2c7fde1f2,"" +0735b01a-c65e-4d59-9016-5e3bed1fcdcf,"" +0785f996-ee5e-46f3-8966-107235de4cde,"" +07906572-63db-4046-8dbf-1ae866664f41,"" +07d441b6-4e92-4d6d-af4b-7f38471fa45e,"" +07dfda29-5378-4a66-ac90-fcb7ec6283ca,"" +07f5fcdd-3aa0-4ccd-9122-f9686886dbd4,"" +07f95b0d-656b-4830-ade2-864e58ac5ed6,"" +088cdf48-9645-4dbf-b824-8836d6a83d24,"" +08d37a26-1ec6-4faa-985a-10a9d0ee54a9,"" +092b1b96-4ad3-4716-850e-8056ccd456d8,true +094a486e-7e50-4717-a0c3-07a456a4cd5f,"" +09541c80-54b4-4bb0-8776-37745a9ac36e,"" +09941e9e-ff5e-44e5-82cd-4c3865e39c01,"" +09ab1f16-7bff-44ec-aed5-8ba400930bbc,"" +0a005143-4645-4ce4-89d5-9d49dca18ec6,"" +0a40e26c-d0ac-4653-a54f-17e23ed9a0ce,"" +0a6c1262-dae3-4ec2-b911-0d1caca68302,true +0a7887db-203b-4a3a-b5fb-cabe9a886074,"" +0ab96751-edf1-4941-981c-c6bdcba75380,"" +0ae172de-1755-46d4-8729-d3acc96ce82f,"" +0b6ee068-26bf-4f1b-8423-30f1c591861b,true +0b95920f-7c6d-4b30-89da-51807a6aa557,true +0bbc7111-589b-460e-a742-20f2131c9c64,true +0bd78173-c348-41f1-9edd-884e3561ad47,"" +0c4242e4-1c5e-49d3-bf57-3810f8b88d56,"" +0c93a4a8-3b59-41f8-a7a5-22a9691710ab,true +0c93c94d-b626-40b2-9561-52168c4961ca,true +0cbe226b-6432-41c9-b3dd-0a14c6d54d7e,true +0d1af4fe-50f7-4d8e-823f-192f8a08ec8a,"" +0d7a0166-7fb2-4e1a-b1e5-b03a53fb5602,true +0dbafa74-2f5b-400e-b27e-fddeac8c28ad,"" +0df8e035-a226-425d-9684-a12d0c58b844,"" +0e03654b-d81d-4a62-8f53-b479e8ce3edd,"" +0e11e8ea-dc1b-4e8f-b809-39f0f07c7499,"" +0e8063ca-3601-4741-a2c0-c0381cc6098b,"" +0e92574a-cc09-4564-a2ff-1cdbc89d210c,"" +0e99ca14-e99e-4394-b962-59c66427218d,"" +0e9cc169-ca91-4201-a78f-1b0f724507cb,true +0ed9ee59-d968-443c-b7e4-c3732ec666e3,"" +0edd2d17-3ac8-427c-ae05-40c99b401484,"" +0ee77005-b9a1-4c51-a599-bd233897e431,true +0eea9efa-148d-41d4-b05a-d39484526333,"" +0f13292a-a654-4473-b78e-c8f1b3328ff2,"" +0f67bae5-3bbc-4c7d-84f4-34583758335d,"" +0f68a16f-1c70-4d73-a9b2-fb384585526b,"" +0f7373d2-3052-4c98-9c04-7074250f44cc,true +1016a9db-50e6-4c17-9e93-18a97b67269e,"" +1037da65-9451-48d1-a35d-bfcae76c2c60,"" +10a7d4aa-af9d-4d03-8d1b-bb53ed123f62,"" +10ad7a5d-e26a-4d4b-b063-239af96b0e87,"" +10b85b80-f6fb-4473-a3a6-266f610dcbc6,true +10cd7171-38a4-4276-b911-bcc8092eff7c,true +10fef9b1-6bdf-4eef-b6b5-e12504e8fc7d,true +1104d152-e95e-4a6a-8225-ab200e4530b4,"" +11148ea5-efc3-47ad-8672-799331f8e491,true +114ead73-4920-48fd-847d-f9aa1a99cab3,true +11597b54-9453-4ddc-9171-2be17df7a511,"" +1167e4a5-ce86-48b0-8b96-6d70125e2f9e,"" +11a0c4d2-8908-4c0f-8e51-bf4be1605441,"" +11af3d82-bb56-464a-9203-8714c270f4a5,"" +11cc59ea-52cb-41e1-97e4-7ed651037442,true +12254e32-b86c-45dd-93b0-c8cbee619a14,"" +12257920-12cf-4945-a72f-80551181d7f1,"" +1266203f-94c9-43ac-af02-a0b8db8a93dc,true +12689eb6-84ee-4d28-9c65-3f3e305266f3,"" +127b14bb-027d-4380-932e-ea34c76592d4,true +128cf2ff-4f97-4039-983b-72e2dd3267aa,"" +1333fbed-c09d-4622-b22c-c67dd32e4583,"" +133a1365-e43c-4d92-a6f8-8c3947b13b65,"" +1359fe32-3275-4c1b-ba16-9dc903334044,"" +137d6ce5-93fb-4201-8bf9-0e97e9595b77,true +13bf741f-7ad6-4a14-908f-cbfe18651381,"" +13f4a39d-ec9c-43b4-958e-be28a97972e2,"" +143099ce-8738-4ccf-85fe-63e54943c7f6,"" +146000e6-9926-43da-9322-de3bf31e8142,"" +147add3d-eba6-4cea-bbe9-823b44794b9a,"" +14a9ad6b-b7bf-4cdd-972c-bda91e5066bd,"" +14afc7c1-89c8-4740-a932-a76d579e98c3,true +151df048-c17f-4a51-83a9-3de5f99be125,"" +1520180e-fb4a-4df6-80be-0b3d559c1049,"" +15268027-ae33-4f5e-9c2c-76327ccdeb44,"" +155d0003-f37b-45de-b34c-d611c2a2cf35,true +15821e91-ec03-4e76-a909-d2f2a14c3219,"" +159c1f96-038f-4722-9e36-98f4dfdb6b66,"" +159c3713-0d0f-499f-8ff6-ecddd44b958a,true +15d9442a-2018-4390-beef-55fd90cf6ea3,"" +15e70015-e207-49dd-93a3-7c0ea5cc456f,"" +1631e3a1-0eb3-4d16-a7e5-b0d8a8a9e630,"" +16af45f4-69d1-4190-9bca-b309c17bd865,"" +16be31b9-a434-46e2-a481-459c5584c907,"" +170e5f14-b29a-4aa8-ba71-4bf381a38d1c,"" +17505b4c-c5e8-4acd-9d10-fb001190a8c5,"" +1795e970-0ce6-4d90-b198-32567c690869,true +17d74e33-f6c3-4b19-8708-4110a2f55245,"" +18a57e28-9c23-4e73-9d90-3694f2e373f2,"" +18c6a217-c947-4e97-a764-c43f33a409fb,"" +18e59ddb-b9b9-401d-9239-0d2cb81936c8,"" +18ee3e63-a8a2-4004-a313-3e332af8a173,"" +190f22f4-5fe5-47e5-974a-fbbe63a676c3,"" +191232f1-58d7-4c4a-93a9-10aee15e5cea,"" +1913a538-60f6-4510-9a76-2466896d2a27,"" +193a64c2-63ea-4677-9302-b56e3f0531ac,true +19a30860-10da-4a1b-abdf-0b03d7b8f370,true +19b01508-db72-4c22-a5b4-4e015a3207f9,"" +19c965ca-c1ae-4eec-816f-73903fe43aa8,"" +19d27e58-26a0-48ca-a199-88d48894da5c,"" +1a3ad6e3-cb5e-483b-babc-ba7d34ffacf7,true +1a7b0249-5aa4-4b0b-adb8-2c958e137d54,true +1ac6d3a1-d1a6-4b0c-b405-0aaaaf37c7db,"" +1adf64f4-1fb4-47f9-a8a3-0ebeb3e7f414,"" +1af4a596-d0c3-4f7c-a217-b83a0755c3b7,true +1b091e60-4c4b-4fb4-b9b8-ee53bccc9b52,"" +1b251a42-a224-4a0f-9d36-44ecc8902d08,true +1b29b772-b873-447b-8aa0-5dd5e9ee1b80,"" +1b614d48-0af0-4d09-a6af-4f3af065da14,"" +1b785bb6-a40c-48e6-a464-f8022ea3cdcb,"" +1bb3d0c2-c6de-42ec-bd60-edc8e3ccd092,"" +1bdd0326-987d-445b-9514-2ea30d948093,true +1c01b494-d0c1-4a99-b110-9de392bbcf69,true +1c120e92-0385-4120-ac83-1b42191597b8,"" +1c32d0e8-7312-4821-aede-6745a83ea45b,"" +1c51f4a8-14a6-48ad-9821-c6408dd0cd33,"" +1c52440e-0063-4170-9cff-a3caaf7c63fb,true +1c6dd7c8-4942-498d-a369-65fac6a837d6,"" +1c79c271-a128-41be-9537-1d744f4971a8,"" +1ce6a20a-2413-470f-86ca-2dacd8811e72,"" +1d9b87ab-8b41-4184-9519-fa9822f48aa1,"" +1da7f725-2fee-4f33-9a89-96253d2449fe,"" +1dcdb77f-b0e5-4b2f-ad4e-d446a6ffa079,"" +1ddc3c30-fccc-4e5d-a9b2-e09ac4f64dc0,"" +1e2bcc56-3121-4b1f-b9c7-7e5503b867db,"" +1e5b7df3-5e81-41c5-b096-d9b5409268ae,"" +1e75c4d7-b570-4339-8c0a-06bdfc9bb485,"" +1e84c68a-4e0e-4980-adea-e46ae2506896,"" +1ea31cfd-f340-4c68-a547-b76365abd1bd,"" +1f170771-0d08-45db-a38d-da3c539b46b0,"" +1f182acd-6bb8-4031-aa86-2f3dd83b4cb7,"" +1f3e4d9e-1fec-4a71-98fd-4912ab64d3d2,"" +1f6b8ac1-b12c-4eb1-a158-25b15965294c,"" +1f8ad9f7-f796-4900-b10b-365fe442f2e1,"" +1fb81c48-c8d0-44e1-9b9b-df53ad7d9456,"" +1fcab6a5-85c8-41b7-a820-fe6e2d84eaba,"" +2031d9a4-3e03-47a2-ab74-6fe7f2fa4ba1,"" +20d47a7e-f7f1-475b-a9b5-6192f2fce0ca,"" +20ee5488-a11f-418a-86a9-cb981c2fdae5,true +2123f57e-0815-49e1-aa34-994270134f3a,"" +2160f03b-4276-436a-8278-300bedd07f7b,"" +21936507-6a38-4d80-807d-0d7cb5ea0311,"" +21d2fab0-5092-411b-b752-4859134a7aab,true +223cae66-3049-48bf-be7b-cf027c415743,"" +22440788-7e64-452d-9025-9a25e4d2be60,"" +2245208e-d17e-4bb7-9bd7-ac75ac0ba134,"" +22462877-9663-4c06-bb5e-040d76bb144f,"" +2253a94d-e511-42bf-a17c-599ae2d5cf0f,"" +226d433b-9abf-4e98-bf0e-47cb3a37b41c,"" +227d84f4-ec92-464a-b3de-f596866ce825,true +228b1b4e-f5e7-4e33-91fc-b7cbe6979102,"" +22b95566-7a4c-47b1-92af-7e0ecbfee6fa,"" +230b746a-26de-434a-9774-54db1ad98a7c,"" +2324dacf-f221-4edf-bdf6-924449cc916c,true +2381d397-0068-4d6a-88e1-a0257d81a323,true +23831569-6aea-4071-820e-6eb5440acc1f,"" +23b4fe3f-62c7-4439-9364-247ceb9251c1,"" +23f91a73-1b46-414b-becb-918e84177886,"" +2439d9a3-6c58-4ced-a82e-27908ed3df94,"" +2450caca-d74c-4610-ac24-1fae13801eb7,"" +24d8f04b-f035-4a9b-b03f-75be9e244ceb,true +25085ba1-43d7-4a60-82ab-e15c477313b0,"" +251e7497-4a55-42fc-8e6a-32febded738f,true +2534b076-583b-4872-84e3-bc96bdee5ef0,true +2583c0bf-6a7e-4100-a5c1-d249de6ec56d,"" +25da1be8-51c6-41fd-b48d-5227e804ebfa,"" +25e1a428-ab0b-40f5-ac25-1e9899458ec7,"" +2606b58b-a924-4128-b3a4-5e4b91deb692,true +261583fb-e2d0-44c0-a677-00bd89a3450b,"" +26344e09-20c8-4405-b674-49638169a7a0,true +266c347b-fa2d-4cc1-a2fb-fe1ba20fbc2e,"" +266cf373-c41a-4cc3-8600-c165a4349a6a,"" +26b4001f-2917-4bf5-bcbf-1ce9dc258171,true +26fb6882-7f3e-44da-9100-58057c9aeec8,true +277e09d4-cb89-4748-b9da-39bee94df8b3,"" +27a75a54-d318-4026-bd63-c80f73899376,"" +2803a9d2-2555-4bc6-9718-7a9dcabd6ce4,"" +2803acc2-63bf-49a3-b4fb-0dd0742fb816,"" +281d5649-179d-4c8c-9a0a-d1159a5df8f1,true +28503262-5aab-44eb-bf51-6fb7534c25ab,"" +2866a0de-36ab-4ec0-a1e1-cce5c697ef85,"" +286decfc-83d1-42e8-9a2f-880f145c71b3,true +2875fd27-8c5b-46ec-91f0-ef3a7a7727b3,"" +28aa1696-dde7-40ff-bb7a-6c5b614405dd,"" +28e8966e-ba9b-4499-bc9c-7f51222e15aa,"" +290440da-118e-4e8b-8d0e-d50043c24eac,true +292aa324-379d-4b82-9750-4c8e23587f92,"" +29506f88-302c-4264-81b7-cf762de48615,"" +29546a71-e5a0-47ae-96da-421275c2cce1,"" +29676c08-88c7-4b0c-82c9-5f6d7dd51f68,"" +2974e695-f2e6-477f-91e4-da8eb654bf24,"" +298120fe-5dbb-4d99-902a-3a6783fc5b6b,"" +298e5099-df2f-4381-8259-b10d0b8c0570,true +299c851f-3566-405f-a217-84c0703d859c,"" +29b17fc8-8e80-4d7d-93dc-bdba1f2cfb9d,"" +29bfab6a-d760-449c-9198-38e8c63574f2,"" +29d4d79f-8a85-490c-b52d-aaf3bcd9a100,"" +29e54cc5-b484-4818-9c3e-08074812dc7f,"" +2a0d7eba-2dbf-4566-af0a-c16161a8b189,"" +2a133e8f-4dd9-49fd-9e3f-5cf122294091,true +2a67b2c1-18cc-45a0-ad53-825c73a3aa15,"" +2a71ca62-7f39-4c9b-b3de-6dfcf299298a,"" +2a7f0959-9d50-4ccd-88e1-6635f2502905,"" +2af24d01-6215-4c7b-88b4-6cee45a156f3,true +2b265cf1-0763-4922-a403-e6f644ed95ba,"" +2b41d0b5-97d4-4a80-bf65-b62ac0cb530a,"" +2b4fd497-9f28-4c87-9b13-64f0d00c0cc3,true +2b54e31d-4af9-4071-ad49-60cae644ba39,true +2b60fba0-0143-4f31-8e4c-27236c7f41d8,"" +2b927027-35f1-488b-bb5b-127a8985cc5e,"" +2bc92150-c490-44da-a925-ebddec5cd953,"" +2be63080-a881-43df-95b5-e371df93c6bb,"" +2c018134-14c6-425d-8ab4-bc3974402b05,true +2c3b614e-7860-47ad-9320-11c5f45d27b0,true +2d358911-c5eb-480a-a9e6-a6fab52ad7c2,"" +2d45c298-d386-43ab-b3e0-ba07dc527a68,"" +2d6cb13b-fdfe-4219-bc13-7de385998bcc,"" +2dbd4567-e6aa-4e1a-996b-af8384f66b37,true +2df38530-e49e-4ff9-93a3-816de29d9bfb,"" +2df89473-f746-4086-a2f3-8ee2d4f63271,"" +2e0bc41c-fdfa-42ab-abdb-28f13cf2ca20,"" +2e13cb39-148f-4299-b74a-0a8e10092e9b,"" +2e45b9b5-8d45-4994-aaba-1fab72c77c5d,true +2e6b11f5-cede-4152-bfe9-523b030a2bdf,"" +2e8c312f-53b4-4cd3-b6da-653c34945d76,true +2ea2af0e-343f-47af-b0fe-6dea108836be,"" +2eb09ab4-c554-4536-a3ce-2979cfb988c6,"" +2edee289-e99a-4a53-a067-f83d29db4c2a,"" +2ee72185-ca8f-40e3-b82d-30408dc8c6f2,"" +2ef91918-bdb6-4a6e-9c39-41d8c513b552,"" +2f1bbdc4-beaa-4d49-8791-6548094c7291,"" +2f24078d-9646-40fb-908d-2628d555dcb1,"" +2f260daf-8fb3-4fb9-a435-670ec8f0180c,true +2f30832d-e6b8-4946-8cf4-0d8da5c359c0,true +2f33b981-f22f-4983-b347-f81d7d72c50a,"" +2f3f7f7a-ba31-4c1e-8a23-c7d1eb1f255d,true +2f5d7d1b-847e-430c-988a-40028f9f595e,"" +2f8b35f6-2b55-4ff0-8d79-5de7805d6da1,"" +2fc9f8c2-4a1e-4e8d-8d91-a6c3b3937e8a,"" +2fcfe30d-dc47-4eaf-9df0-7943a2c37fc5,"" +2feac451-2301-4ed5-8bca-baef62790656,"" +2febeae3-eb61-4743-8ff9-42547a77a40e,"" +3006707b-dcca-41ce-b138-5d62e6b56984,true +30371578-4ad0-498b-a491-4e194c7466fd,"" +305d5449-57c6-4362-83cf-55b4dae133de,"" +30ba9f80-e595-460a-abf2-b8f46a62e606,"" +31504825-77aa-4a0f-a634-ca806bd886d3,"" +3169cf91-33f7-4b7e-b49f-d4b884760685,"" +319dfae4-0305-4520-8ad1-9679383cb74f,"" +31acc736-076e-426d-8a32-426d91bf6511,"" +31e26c46-af6e-43a2-8e27-a8342f53e817,"" +31fd2352-7eb4-4ce8-9dfb-b3d3a1410c82,"" +322637fc-2d75-4a8f-89a6-a4c963869112,"" +32596eae-6236-4b3a-99fd-e50856971f59,true +3286c996-861f-4fe3-90a8-86ec826cdf9a,"" +32bed860-5df5-4bbb-8bd1-c95c016561ee,true +32d92e2f-d02b-4006-bf16-d1c535948e80,"" +32f3d9fa-6c64-49e8-a6dc-fb51679753e5,true +330fde18-0a86-4e67-850d-fe1c8b376dab,"" +3322da98-67e0-445f-ba1f-225eb7d297a3,"" +33333f69-f8a6-4def-a5f2-bab5a26bf054,true +33826788-b600-476b-9843-17fbee2cd849,true +33a0b9dd-c88a-46cd-ba8f-af3d4d6cfc27,"" +33ac0c39-1552-408b-97ea-a90c9187d61e,"" +33b22ef7-01a8-4e8f-8b5d-9aef37e46896,"" +33be0567-5c42-451b-bd71-c09f7de30524,true +340dbb93-0888-4f47-b18d-0b86d3e40248,"" +341634fb-2646-4c69-b57d-b5f00ff4c83d,true +3420a3d5-d5c7-402b-a4c5-02e7198c5b7e,"" +342699a7-cb58-4a9f-bb79-306483dfa652,"" +34335eed-f6c0-4e05-ba6b-1ecaa1fc121c,true +346734fc-4798-48e3-8880-a603d52e9ca9,"" +346a282a-439f-48ac-875a-cc34643d70cc,"" +34a83c04-2ea0-4638-ab6a-0a4bb9a3ad4a,"" +34e44ce8-26fe-48c3-8fe8-5cd8f260aee0,true +34ea288c-f987-476a-a8ed-0ab2c1b92833,"" +3590f230-1be0-4c7b-afc6-5bffec9933ad,true +35cf6113-d23a-476c-b18d-f86fbfb3ea2d,"" +36160c58-889f-4543-bf04-dd29cac9ce19,"" +361688f0-e909-4c9a-b5d8-10727c9a18fd,true +365ca17e-d5b1-41d2-982e-3ca7afdc0a0d,"" +3669a209-82a0-4f9e-8026-43cebad7f3ab,"" +366fa6b3-9595-4624-ae30-20257ef4885a,"" +36957f27-97d1-483a-a8ea-bd5b1484ad12,"" +36c9ce76-e047-487a-837c-faffdb2676de,"" +36cc6703-7757-4c6b-91a0-a43055c19bb6,"" +3732940c-2fa8-4301-942a-e58fd8d73421,"" +373609ea-276f-4b96-9c94-d96ad532246b,"" +37770a6d-bb26-4499-a6e6-2eb399a8ad07,true +3793c189-6e61-4f1f-bb19-a01a306267d7,"" +379779f8-dbce-4dfd-b18f-ce0485b0d6aa,"" +37e1570f-2c29-418a-acee-6688b6ab4279,"" +37ea3114-1b65-4c47-9e65-4cbcf339f8c5,"" +37fa89b6-2f6e-4985-b733-1ee4b2fd59e2,true +381dcabc-52cf-42e9-9624-d4e1270518a9,true +383aa480-8db2-49ed-9542-c738af90f344,"" +38898dca-756f-4267-aa1d-1fe52a6e4749,true +389079a9-b902-4212-a8c1-7187e818e016,"" +389a8f10-ad75-4d27-9d11-2c4ba0dc55a6,"" +38c329c2-4078-450f-ae85-e411b80f30ce,"" +38c7c98e-0372-4892-aea0-ba21ed298958,"" +38d59ef7-b207-4d55-b74a-a1da3d0c10df,"" +39d3bc28-0ce7-4d1d-9adc-bb9933568809,"" +39d7db36-7baf-43d7-8a0d-970ae00456d0,true +3a1941c5-c6bd-45b7-b306-d2a1a51c5eab,"" +3a8a0b91-a492-42de-9b64-fc552476d8ff,"" +3aacaf9d-af9b-4220-a836-1d0c3cfd5245,true +3ad5553b-58b8-46c9-8023-e1acb85be08c,true +3b4bf6ba-0f27-4228-b6e1-c16dbf82e5cb,true +3b7c654f-9a67-4402-a571-a5b0a61559ab,"" +3bae6ece-f05c-4e3d-81a7-64f10f907bb2,"" +3bb8d5a9-87a6-4155-8f3a-7a0c841672c1,true +3bbe3af0-0f3c-41ee-9c41-70d89b411852,true +3be31b98-5218-44c6-aa92-1e054793d9df,"" +3c496888-2ead-4d5f-afa4-8ab39b57ca03,true +3c504a58-92fd-4b2f-a308-21a9588b1850,"" +3c8d926d-3156-464b-b385-192257411798,true +3d2c40b9-2ef8-46d7-8215-bb898170902f,"" +3d87abf4-a569-49b5-962b-32c11816a53f,"" +3dac5f75-f71c-4092-b87a-7b8434af633f,true +3db4a850-ef45-4fa2-8eb1-b1cba4477d8d,"" +3dc0190f-39b2-4a1f-87d9-1ef41a73fa75,"" +3e4206fe-8856-46be-98af-d6fecd52db92,true +3e58093d-c387-4a07-85c2-41a519d0c398,"" +3ebd1cdb-5546-4dea-ba79-a88aef933640,true +3edb8e33-d674-4c25-aa75-7303da0229b9,"" +3f5c46c8-4a8a-4575-b169-6bfd5d752d98,"" +3f771434-cfed-4f34-9671-0ef302b5f9e7,"" +40100d97-ad9a-459b-b044-66568d36c0bd,"" +408159a9-8c38-4cb3-9488-d83331f1762d,"" +40a1dd3f-6ffd-4463-bf71-42057ee61689,"" +40a75d7a-909f-4657-9ac1-5b56df539129,"" +40eec0cb-8f50-4f61-92a4-7682c48511f9,"" +40f3ae1e-4855-4d78-9fe9-c8348058088e,"" +4115b0b4-f738-405b-9351-ae11ba4bd07a,"" +411cbadd-c2d1-4fb3-8ddb-d8bcea3aa176,"" +41496990-adc5-44d1-96b8-15010bdecc89,true +41a3cf00-0965-4c17-bfb5-247a7da43699,true +41a86c21-2ea1-436a-86a3-9173c7d95d9c,"" +41de6042-e95f-4bca-ba69-f55293248f7c,"" +41ecc0b6-ae98-40e0-a836-9d26070e248c,"" +42187018-d27b-4831-913c-575d07e33c54,"" +421e31e1-5ffb-4c6e-957c-728b6218d6da,"" +42350e13-ab61-49ef-aea1-802f6b7ea955,"" +423800b8-5a89-4eec-a12f-451c533fac50,"" +42895609-81ce-4cef-81db-069efe9a207d,"" +429c5342-c5ae-4c55-98aa-e4f7bf0b4670,"" +42ac25e5-8828-49bf-bb5c-a6001966c6ff,"" +42c5e47c-ede1-48eb-bc82-2ed835d67495,true +42fe77c3-239a-4afe-ad76-8c221b543e32,"" +43179e01-e94b-4f83-bb60-60f181388048,true +432bf1ce-0e8b-4621-b998-819fee53f80a,"" +43522eea-96c7-41f6-ad41-1c190cece6aa,"" +4358ee55-cc06-46a0-8b44-6b89d09c380f,"" +4380d762-baa4-4c30-905c-59b33dbd793f,true +439dfa4f-917f-40c5-9313-0d5dcf7ecf2f,"" +43bdfaaa-7474-4503-ad93-730d777e6db8,"" +4416e165-8ac3-43e0-a9e4-6c3e789ffdd2,"" +44233ad5-4eb9-4709-9c6c-e7797d9c9f1d,true +44237e1a-31ca-4635-9ff3-e3c71f587972,"" +4442a5e3-5793-43ae-92ab-f2d87a70cc0b,"" +446f7fac-2202-4149-9860-fe48335349b3,"" +44713d16-f673-456a-881b-b0526ee36651,"" +44b741ee-bdb2-4732-b484-e435854de348,"" +44dbff22-7cac-4f70-886f-b8ac2dea7130,"" +44f714c0-88de-43a6-b69a-4d62d8fee57e,"" +4522ef75-31ae-43e5-8e5a-5caf5c83fb35,true +453cd2e7-723b-4e1d-b414-f46f9152edcf,"" +45467cbe-cc21-4822-94f5-82e1203056ad,"" +4546f579-74ab-445c-8619-431f60a6efdd,"" +455f8ebf-b074-4035-915c-91cd15380d72,"" +459f6da9-0eb8-4b26-85b9-a8f63c587fab,"" +45a306de-3731-40c2-8bd5-2c0fee3b9f99,true +45d45783-5a68-4c06-9b2f-88b7ea1f8347,"" +461bea3d-934e-46fa-a112-9a761140372a,"" +4672ebd9-08d2-43fc-ba92-ce13b587af3b,"" +4675fa7c-6ac8-4bef-8756-88f757370a10,true +4680ae70-6d17-40fe-b55a-c3152ce8b246,"" +4687b275-d990-41cd-a146-2715079a8ecd,"" +47014b4d-07f1-4bef-9792-82c027525c0f,true +473bf60d-dab0-4531-9903-95dc6680a660,true +475e95a3-6d57-40f3-a0af-9daf1b35618b,"" +478fbabf-520e-48a5-b3a1-a3e535d64345,true +47971ac1-1e72-4788-ba6f-bea608e59076,"" +47f066d8-abf5-4064-84fd-dfb05649e0fc,"" +4808c4d5-7733-4601-8401-632bf5924c06,"" +484a19b7-9816-4a66-8f0b-51cb5f146818,"" +48693756-94c2-434e-ad88-e573e5278a6e,"" +48fae21d-0ac1-4e4a-93b7-ac4ba7c2a606,"" +49088daf-d19c-4bab-9084-c67f0fba2c90,"" +491118c7-998f-4d45-8b58-da724ac011e1,"" +4925b797-e285-4d02-9aed-dfac19b0875b,true +492f6c9e-3e1c-4043-aa08-03c610da33a7,true +49466fee-df56-4b9f-898f-96b65318d833,true +495bdeba-9ab9-4756-b4e0-a4aa8b039a02,"" +496f0696-cee8-425a-81d8-1df8fd17e4bd,"" +49cce532-3ba4-4db6-b518-ea934588bfae,"" +49ee8b11-a7c9-455e-9a9b-835dd617e861,"" +49f41136-fc63-48a8-8ee2-b7536d3bf0e7,"" +4a6eb9f9-49a5-4d36-85dd-78512fbe593a,"" +4acf651e-af55-447d-bf0d-9adaef4cf6bf,true +4ae0f855-534f-41dc-a4b0-879c771e3379,"" +4aeca194-6d0d-48ad-8fd4-6aea3be783fd,true +4b322f4a-fa06-4509-9783-dbbf12960b77,"" +4b68a6bb-4fca-45ae-bfb4-b814c67bdac7,true +4b86609f-db97-4a0b-9860-3ce519072521,"" +4bbf3839-bd13-406d-9f60-d9eb9daa4dcd,"" +4bd9e2b4-e195-4a83-ba1c-478e66a7bb5f,true +4be334e3-94a8-4223-846e-39b5877e9ff7,true +4be59cbe-d89e-4abc-aca4-410a7e8cf580,"" +4be72d80-9418-44de-ab47-e8f2f2b9f2e4,"" +4c9a28de-ab9a-4e69-9ca6-375512c2f149,"" +4ca8a902-7ac4-4634-b1df-456175c39d14,"" +4cef34bd-c677-4647-b940-840a6868cb9c,"" +4d0e9887-c81c-4ea0-9026-41ceb9ec9a51,true +4d45564e-e14c-4289-a637-06193bfcef2e,"" +4d79c0f1-72b2-4e94-addb-9adce3f49c2f,"" +4dad93cd-b188-4750-bdb0-4a5501568d81,"" +4db059c5-c4d2-440a-84ac-360312e3c590,true +4dba5ae8-2d30-479e-b831-80d39c396280,true +4de45bfc-2a41-4e0d-a0e7-c6caf6ca0f24,true +4df7ae55-1947-4d89-b51a-496b34c4250b,true +4dfadd90-3f14-4ee9-abbd-a348a5d6a7d9,"" +4e0d77a5-782a-4b03-8867-f51a9704368f,"" +4e74bab4-e328-447e-84d9-a29bb7fa1548,"" +4e8a0b34-71b8-4da8-80f9-f015782b182c,"" +4e9a316e-406b-496b-acf5-6eec80f25ee7,true +4eab3a46-0ea6-431a-8c65-5b5bdee521cc,"" +4ec34b7a-8389-4d01-bca5-06e678089d41,"" +4f147e12-0e70-4ddb-b4e9-4ea25361f290,true +4f18952b-be4a-4d77-ad4a-05276a6bce1d,"" +4f3f5836-8683-49cb-b5e8-a9881796a202,true +4f62ec59-e478-40ef-8617-e6ed599866f1,true +4fe2d379-ba65-4ee2-a584-f0bc63a8523e,"" +5005cfbc-b57d-442d-976b-34164aeb1977,true +504859eb-2ed0-4184-99f1-79f436fcd692,"" +5085d62c-dc5e-4f25-8f0c-700e57bb71fb,"" +508de40d-3c1a-4bfc-8713-11dbae730004,true +509d7086-3120-4f61-bc92-4ab924f30ffa,true +50a9341a-67f3-4d18-9118-373db23eea3c,"" +50b76e4f-6575-47f6-9e15-a96b58c492f3,true +50dbd55a-e4db-406d-943e-7624d96c554b,"" +50fbe67f-8d29-41a7-8cc6-3df916cd922f,true +5105a245-dc96-483d-9a0e-52b39e664d27,"" +5106bb8d-43a4-4783-97c9-42d707990a2c,"" +51164c4a-fa5e-4246-80e4-a01e1d7707d6,"" +512a1f5b-9f83-4420-a4e4-978e33432dba,"" +5253d689-c7bd-441c-ab16-3721f1e3b5cb,"" +527721f5-443d-4367-84e8-0f15982e7587,true +529479a1-36f6-4133-bb4e-2f3cf677626d,"" +52c03be3-5715-4edd-9809-ebff7e2ba57c,true +52d51fd3-d0c8-4046-9ec3-1a3f523d0b75,"" +53010db2-bba4-4751-9f12-f51429c9c558,"" +53339064-e7f3-47b0-a5c6-ce77d50b7489,"" +537bd7f9-9f4c-4bf1-bc21-9d09dd655a13,"" +53e5edab-cd6e-42cd-b6e3-a36d57be3685,"" +5441b3d3-b8e8-4e36-84c5-623e9b199daf,"" +54487624-db9d-43cd-b4a6-0b4d3934f7bb,"" +54627df6-151a-4f3c-9fea-374217afba2b,true +5496c32e-97eb-4f6f-a7ee-46bd2123f966,"" +55054683-a9bc-4c84-b1b9-6a1540e1168f,"" +55449cc5-a88a-4181-9550-5d2bb23a293e,"" +5565df4e-b483-407b-8816-ca9e8ddefc27,"" +5578c18e-84ae-4f7a-8cb6-ce285e1b1ff4,"" +55936779-3082-43b8-a1e8-a8329bf19a8c,true +55954c84-4a6f-4667-b070-5fa9185fe3d2,"" +55dc6b95-e01c-4812-a812-2cbf20de74cf,"" +562f66c7-19ed-4d05-a14d-6aaa37e6eb16,"" +56617f73-b5fd-4e8b-8bd5-467af440f0a9,"" +569f7616-50c5-47c5-94c8-fa3b3097bc97,"" +5703a2ef-ccc3-4ca0-b100-a1c567f1dfc4,true +572e67ff-4f0c-49c4-b55a-36a958d3e02f,"" +5742e203-dd0b-43c6-9dfa-304240238737,"" +574873f1-84ed-4171-a51b-6a2c609e9afa,true +5760bdfc-cbd9-4763-841a-bb31a972f551,true +5781b9ed-eeb3-45d3-a364-43cf8ea348f6,"" +579a0bf9-87da-4887-9700-a5098cd7d720,true +579eb155-01ea-4066-9dae-9ddce5ea2041,true +57d84d96-a1fd-44ec-8460-588b477502a5,true +57fa04a3-a8e7-4699-9115-95957be21dea,"" +57fec813-6327-4290-a2e2-ca74a04cccf8,"" +588b882b-0bff-427d-8b34-294f2fe55189,"" +58aa2384-e80d-4083-a173-ba534f152680,true +591075a3-92bb-42b7-a6b7-d761add1ad2f,"" +598288b2-fa85-4bf8-9e01-2d0e86711aa7,"" +59857d17-d8e0-4932-b92c-97bd36a31d3d,"" +59a25b17-fe55-4592-abfc-0c22e237bbd9,true +59b131b5-2db4-4877-bcc1-2206865df718,"" +59e29c4a-102b-46a3-a950-3babee7cfccc,true +5a0eb3ab-86ce-467f-b389-94f026fe0f7f,"" +5a4bc06f-8ed6-4cbb-93f7-5c5faee7e855,"" +5a9300fb-d831-4b67-88b2-843d2eca4441,"" +5b0954c2-a2cd-4013-b866-a6e0474c12c6,"" +5b2730a8-6a24-4bf7-aacd-f56fd380c696,"" +5b44153a-1ea5-45a1-9d58-bbc0a474ce1d,"" +5b5f6d2c-1899-474a-afae-a98762630b06,"" +5b72c7eb-4584-45c0-9039-5fc762b1e50d,true +5ba15ba9-7112-45ab-b7e5-353d52af5fe3,"" +5bb09da7-8ce1-4642-a113-fdd1fbc73897,"" +5c085687-51d6-4cbe-9b71-0a58718c1d24,"" +5c17c3d7-a7ba-4756-9991-97bc0b3a46ad,"" +5c48ba78-4e71-441d-b03e-ef305d9e2bde,"" +5cce02c2-3537-4d34-b085-7cf145590292,true +5cd558f0-20d1-40c7-8197-5a41ef97e634,"" +5cec04b3-e52d-4a6c-af98-6ee5d104b20a,true +5cf73976-33ee-44ac-aade-5c64e56d7cd9,"" +5d54c744-365a-4db7-9675-79d64bdf55a6,"" +5d5dad82-9b58-4302-a494-02f44c3e94bd,"" +5d790c80-5b87-4802-9b89-02f949846f39,true +5d883f31-f417-4d8b-bdbe-2cf73d1aa4b5,true +5d9e8c9f-8c95-4ca2-9de2-2cf875868ff9,"" +5ecf975d-fa2b-4c6c-a5e6-fed730c9e035,"" +5ef7c113-d5bc-486c-9be9-aa39aea779d0,"" +5f6ad8e2-eed9-4b2d-8697-4e2e742ee1d4,"" +5f92feba-5842-4894-907b-9ceaf36d0f38,"" +5fc1d33e-168e-435c-93ae-c619999b6f9b,"" +5ffb50f0-0963-43d0-b032-3e6b2621e1d7,"" +602f4e55-f69c-4103-bb6e-7d23fd1b06ac,true +6042dc41-8cdc-4825-9ab2-bb22af5c5fa6,"" +60635e6e-3d74-4f56-acb3-be8edc522837,"" +60959eb0-ed9b-4043-b8a1-0b0f27789642,true +60996e5d-082f-4d1e-8cc5-6cc626974925,"" +609a5704-81f0-4b67-881c-d00e87b74c0a,"" +6198d258-17e2-4956-8064-1448f4744303,"" +61e75b9d-46a2-4d74-a0f0-37dc66a09450,"" +6220b92e-946e-449a-a2bd-3241e6a453a9,"" +62346d78-6be2-4cfb-850e-6ca1fc3f5e35,"" +62348772-d9cb-476a-a016-32f16e44cb33,"" +62ad1938-6235-48d8-aafb-2f773a40c6e1,true +62ccafe7-fbcb-4cbf-8584-224cd58ef421,"" +62e4244f-3f34-48f4-b865-c8b1c3ad97f5,true +62f342e2-5499-47c8-959c-4d786aa5fd67,true +62f6ce69-77ca-436c-9d40-3eb541853798,"" +6320229b-25c2-4355-a288-17efb054df6f,"" +633dac04-7a54-476c-8482-9631c351ac26,"" +639dab1d-3282-4de9-84f7-727d0462f11f,"" +63b2b9b1-aaf2-4539-97f6-aa6815e75f21,"" +63d6c1e2-ce14-4e89-9bcc-f3e3e0b6fe72,"" +63e5b7be-525c-4a0b-852f-4cb090178d30,"" +63e709bb-a56a-452b-a48c-5125a367572a,"" +641068ee-0805-4403-a159-4fdb185cfdad,"" +64115066-e659-4cd0-9a56-fcd43a6df27d,true +64148c8b-89e4-4394-b044-27088ab3adc6,true +6433bbab-cad9-4e98-b488-c06e37acf268,true +645572a4-5806-4854-b89c-939463a904f7,"" +64a2df8b-688c-4368-b672-0406ac7032ad,"" +64c6ac23-d619-44aa-88fc-969376bebfcb,true +657aed74-c373-4c45-9de7-b75cdf62577f,"" +65ad0ff6-ba27-4eea-a400-3d96240a5702,"" +65b9a660-2f5f-4d9b-9f15-5e572d65ee9a,"" +65e87fc6-9232-4ba9-baae-61d7af6e54f4,true +66037242-eaa2-43af-98ed-190c3079833e,"" +667d62a5-160b-4657-be40-beeb787d51d7,"" +6683a92d-2d09-4875-9403-b977bc2eb37f,"" +66be2f9e-fffe-454e-86ae-75dc713ee934,"" +678223e6-745c-4d58-888c-b910dbb9039f,"" +67cf1dde-11a3-4aff-a7f2-a8b8288aac8e,"" +67ed7f4f-4873-409b-a04a-b928cb461719,"" +68059a10-b5eb-41c5-bb44-c56ecdb6baf1,"" +68249708-6a5c-474d-b1f1-214514fe5dbb,"" +682b9750-e94b-4522-b277-0d572b67e94d,true +68935870-c60d-44c5-b57a-000c0a99e9d2,"" +68a1d32e-915c-417d-98c9-3154ffbc9249,true +68a84b5d-edb4-41ee-82aa-8612ab7b2b38,"" +68dc301a-6b81-449d-bcf0-7b2f3d3335db,true +692cc2c1-44b0-4c47-86b1-2d3c4a5848bb,"" +6938bd7d-06d6-4a9d-8234-9123aa238788,true +697d10ec-d72e-49b1-b2f9-ac8a3be5c856,"" +698df1c2-7cc5-44cc-ab61-5420d92c598a,"" +699d6871-1043-4581-8dfd-fd1b6e47d518,true +699f0f86-fc10-4f15-9a78-19515e4e4958,"" +69ae7f71-a967-4c7e-a046-cef77b622f70,"" +69c3f870-e1a5-4785-8dc4-067a7590b949,"" +69e71f61-bde7-494c-b65e-346f5972d21a,"" +69fe60c1-d372-41ec-8671-94090b970a7f,"" +6a4adab1-6b87-4e61-a400-dfa4aa4af613,"" +6a50955b-b259-464b-b3ab-9fa173caeaa5,"" +6a8a69b4-f6d9-457e-8466-fdf87ac0585c,true +6a967445-47ae-46aa-8754-2e48915efc2e,true +6aba4aa4-9fe6-4759-9f76-705127be70c0,true +6b8c1da8-2bca-4e16-a04c-a99a04c76d85,"" +6b9af3c5-cc2a-40af-9264-1c68ea98d031,"" +6c18b368-8579-42e7-b0da-3500ed15ab82,"" +6c3be815-1494-4c82-b655-a3e7c92ee945,"" +6c7689e6-db2a-4544-bf19-0b70c6d74279,"" +6ca383d2-551e-4c89-b07f-c0faf9b36dcf,"" +6cac250a-d650-4c9d-a896-b38dd7e72955,"" +6cc22c4d-6014-44f6-a1e9-0b8eaf3f837a,"" +6d2776a9-6add-4222-bb43-64bd4dda65da,"" +6d34e7f6-d885-49f6-b072-6d06f6677a2d,"" +6d564224-4c2f-4bbc-af57-9895ad3b57d6,true +6d5ff978-ef70-4eba-a1db-536985901dce,"" +6d759795-d565-43ce-90d5-91b167b18c10,true +6d813c64-a15b-4e50-af82-c2f18f82a0cb,"" +6d94712f-7565-424b-ad4a-e1789fe2a639,"" +6d955124-bd23-4958-bad6-b36cdad50a5a,true +6d986c2e-b12b-4a6f-ac80-d3bcdaede076,"" +6e02f522-f081-402b-88f2-38e6622bd006,true +6e362fd8-b0b4-4cff-85c3-6586920627ad,"" +6e750faa-cf8a-4f5e-a02d-1471741cfdab,"" +6ea2fcf1-31dd-46e4-be06-592626e212f5,"" +6ea7ff7c-35f3-4350-9f9f-104c5182cd0b,true +6ead6acc-52c8-4df1-b00e-b46e2e2f0a04,"" +6ec2fab5-4d76-4c80-ab2d-577b041c1aee,"" +6f550b68-a7cb-4314-ba74-d7d2a6d56d0f,"" +6f63c413-6f3c-48db-990b-fcc749cc6355,"" +6fb0d554-b14a-4bbb-aa5f-a4d5c082c2dd,"" +6fd78455-d026-412e-ab31-d9e1a02923dd,"" +6fdefa93-f23b-4eab-82d0-1e9ce7c6c17c,"" +70777428-a2a0-4dd6-b048-1ef2a27d9ec6,"" +7084e394-9ee8-4b0e-a3f5-e8d091ab4fae,"" +70a25de3-45dd-4848-a394-e87d4f6b6930,true +70af4c33-277a-4a9b-8022-60d6bb0ac6aa,"" +7102c0da-2afa-4da2-a79b-a82742e4e3da,"" +713231d3-d739-455e-a913-41d5a1dc8571,true +71f73b19-86e6-494b-8038-46a67396de4b,true +7227c5bf-3699-48cf-a0c4-fb8f5d50540d,"" +72619c45-9cb0-4abf-a98e-20cc49a4e4f8,"" +7287ec50-e64d-418f-a83d-0d3d93981f62,"" +728f469e-7a39-4a77-8118-944f83c2cf21,"" +72972472-355f-4538-b69a-23aee8614438,"" +72baabec-7047-49ff-9e67-338361a06e5f,"" +72dd2d03-e239-4a2c-b509-769886057217,true +73364971-4250-436d-b85e-93df377b2911,"" +734b53ff-038a-4c79-91e5-c14340b43145,true +73fe21cd-4493-4afb-ba72-376f75bc2469,"" +7409225f-77a9-474b-889e-38eb4db0da2f,"" +74556136-3dd8-4074-8e26-160ae1b5f780,"" +748bb216-e420-46e3-9aef-6fe6b2020a74,true +74cfd92e-5da2-4fae-be59-1546f5a22e35,"" +74dbc21d-e056-4050-8f0f-04194ac43995,"" +74eca765-7327-44ac-9868-ed01c1dc163d,true +751225c7-3a03-40d5-980c-89f6020bc2ca,"" +752045ea-d278-4939-a82f-3ef29dd7616a,"" +752b713a-e049-403f-9d4a-f560ee0d4681,"" +752f97cd-9398-4830-ad13-38e26e3a8bcb,"" +753ae998-68fb-4917-93ed-484ef1753d9e,"" +75509cde-925c-4645-9a91-c1f244863842,"" +7569ce07-0cbd-4a30-bab7-d5f482a27883,"" +75969f3d-bddc-4e9c-8872-a33f108027a2,"" +75a6eba8-9d1c-4de1-b084-00a5aff10eab,"" +766ee5bf-c1a3-4837-8ee4-0c81e7252c91,true +7690d499-c3d6-4092-89b7-81b3425164be,"" +76a04620-5da7-4914-a5b7-5a8506221f16,"" +76b1d775-0f28-4bdb-b6cc-a2441e8e17b9,"" +76cb0eb7-3679-4104-8121-9e01d36de541,"" +76d885d5-454a-4541-8ec8-61f7972f17f7,"" +76f4f348-f65f-42da-a338-1e1f28df0ffe,"" +770f3f03-8a4b-4e32-8443-49b14d54ad3b,true +7714ce28-a83c-48dd-a84c-ba3565432818,"" +7728c15e-82ee-48c2-90a4-afa79e048a52,"" +77293357-c730-4960-a8aa-c25ea0775c3f,true +773da43c-fa5a-4a21-85c1-724b9d72bbbb,"" +776cf7db-0b09-48dc-b089-2d24410ed4d0,"" +77a645b1-5231-4486-8f16-0470ecc3e49f,"" +77a7196b-7391-4a38-89a1-e01b15e9e2a6,"" +7879670e-7953-4dc8-86ce-88f346cee653,"" +78a727b9-5bb5-4ed3-aec3-a3f804c79734,true +78bfe231-0661-4502-b18d-28c24b223dce,"" +78edeaed-91f4-4f03-ab2d-38b720e5e6df,"" +7914b992-59e8-4541-8250-427de113802f,"" +794a6b0e-1e0f-4c12-ae52-47b53f385a03,true +7957634e-ec90-46dc-9779-fb000649bd40,true +795d85f9-c2d4-4219-8e06-f6fbcef30e33,true +7968bcd8-e538-443f-b4a6-e41b9766965b,true +797ac9d5-9ef4-49bd-af56-1bd0c971467e,"" +797d8c9f-56eb-4487-ae30-5c4dca379bf9,"" +79813005-389e-4671-850d-b9a5fbc2345d,"" +798cfbc1-a7b2-47cb-91af-43af0ae0119f,"" +79a428bb-83a1-4338-a613-9b732f6feb4d,"" +79c5ee51-42c7-4fc0-8e8b-7ce9acb5133b,true +7a281b03-7145-409f-9c5a-c45b6e544a4c,true +7a543c90-f6bd-4343-8311-6818c3a9066e,"" +7aa8cd33-8365-4c53-ad1c-e07dcf2f86dc,"" +7ab27dbd-f952-44d9-bd9a-f67d5790fc82,true +7b4255ec-4801-47ee-b12e-4b874c5eec1d,"" +7b8c6a88-3a53-48bd-9b73-bdc3f087e6b7,true +7b9a52de-eb76-473a-a622-4b19d50b6bc9,"" +7ba6211d-542d-4508-96fc-712328eefc69,"" +7bb75949-ef13-4934-a19b-798f6a5e7866,"" +7bdf88dd-7924-4b93-9ff3-d543a170e0d8,"" +7bf5b1d8-3edd-458a-9730-187ac6738857,true +7c197c69-7872-44cc-93bb-c518b8403da2,true +7c6c40db-df29-4b8c-8881-3f1765e51c2d,"" +7c6e1d99-bc80-4172-b76c-226f6a618b48,true +7c85bdf0-7156-4555-aa37-49e4195ce0e4,"" +7c94eac0-d189-4107-9443-452d124db20f,true +7ce02db8-4ca5-42c9-afe5-b818e42e1767,true +7d4a09ac-d7e0-4675-a9f0-313c73fa7906,"" +7d83ed97-4c6b-4ba0-9dd2-423f59e8bba2,true +7da3d62d-56ee-4707-b036-2c1b5e111aff,true +7de9fea9-9923-4dec-9c83-1cf13b81ca39,"" +7df8f003-7458-4083-8ef6-20a049526c19,true +7e194149-713e-46fb-ab75-4a7cc2acd94f,"" +7e618ba1-1acc-4c2d-9b80-fee3d12884c9,true +7eb4427c-1b25-405e-8ef1-38fabbfb1a0d,"" +7ebecbcf-c963-4cae-9b1a-8fe44e175008,"" +7ec8d0b4-41e3-40c7-8f18-fa10b249e023,"" +7f054bf1-9bff-4ade-8841-ebc67d4299c8,"" +7f35f61b-9551-45f5-9641-c9a74d7ef110,true +7f8f71fd-602d-4ce0-ad32-211f5e7ca765,"" +805e5d13-6a31-405f-b43d-3449d73a50d3,"" +806085ba-3993-48ca-851b-a5dbe0690926,"" +80aca9f1-6907-4658-b0e4-bf6f53838148,"" +80e8b1f5-ea5b-4abb-9a06-b51bcce673e6,true +81155341-1ad4-4473-ac14-9257704203e4,true +8118e76c-1974-471c-9cf9-0b31ed231efd,"" +8131895a-c6f1-4b0c-aba6-cc3a52066646,true +814970aa-0552-4357-b958-d572f56421ac,"" +8177f823-7335-4da6-aaf9-354e05299d71,"" +8197ccde-83c1-4ab3-bf4b-e849667f1f31,"" +81dd2be8-9b50-4d63-bc4c-22c4dcba127e,"" +81f1e8f4-a335-4910-8a27-151141514c39,"" +822dee6c-337f-4a5a-a22b-7de9e64d1144,"" +822e3fd3-ae35-4e89-a5be-feac7c6f62cd,"" +8242d422-fa39-4374-85c7-5d267d68a0a7,"" +826cf99c-9d92-4cef-b717-d5c16ea7525c,true +82e58dcf-ea72-4a50-9eab-a74460d58e4d,"" +82e8a61e-9eb4-4fce-ad43-881202cdfbef,"" +82f09a48-3b8f-4a2b-9067-61b51d85c500,"" +830df854-5f4b-474e-9aef-e7be6b452a92,"" +832e240d-da2b-4658-b638-cd2cf2f08912,"" +83648485-484e-46b6-9a11-64a48fe471f0,"" +836f63ac-40e4-4170-a9f0-f170c4654db8,true +83b971fa-bf7a-49bc-b1d9-09680e8eeb24,"" +84a15a42-428d-4092-9f54-235b55b11de3,true +84c41fdc-0d34-4d95-8999-7677e4235e5b,true +84c90e57-2348-4843-a557-444e3a82ce43,"" +84d36e49-fc69-46a1-a5be-e9842e8e192f,"" +84ed43cd-526c-4055-9422-c31dfefa43ef,"" +84f38578-0070-46e8-809a-65c75623829d,"" +8502fd55-eeba-4c91-8274-4f34dfe89977,"" +857895a5-3a82-45fa-9fc0-00380ef01b47,true +85827568-1a34-42f7-877a-99a2d3a0f5c8,"" +8591ca33-6c89-4352-adf3-8261ce1bf58e,"" +85ea993c-f6d7-4b7f-836c-3c76206d66c1,"" +8644762e-70c7-45ba-a0fb-92fdb9e6c73b,true +8668b20d-ed68-4d67-91fe-d35c385a3408,"" +866bd41c-debf-4a53-9108-fd9c0784103c,true +86786682-7825-48f0-bbc3-3b1a127aa654,true +86859d13-26a6-429a-a38e-be69f1af9b4d,"" +86946982-ff92-46cf-b6cf-caa45ba4a986,"" +86bc9f92-f89b-4fb1-affd-093a785ac1db,true +86fee3a6-5135-4265-8c8c-04ac00ea8e12,"" +87061f96-823c-4e0e-9dde-8034682651a8,"" +87346977-9c76-4d76-b8a7-23ded71369a2,"" +8746d3f8-1943-4ab0-af49-f7d40aad5cc9,"" +875be020-8c24-450d-9ddf-f642e60e62f2,"" +87e5046c-332f-4ab2-b8a9-ab15cfa91fa5,"" +88274e29-0d4c-41c2-9315-f96c93d44c0d,true +886a561a-e64b-44f8-bc77-76724e0c369d,"" +888dfba9-08c4-4048-bb9f-23fe80dfcf5f,true +88a3a7ea-728e-4bbb-be44-73093e564cce,"" +88aecc11-264c-4a3e-a2b7-94c99d39fde7,"" +88cf49a9-ec8a-461c-87bb-003866a4e28e,"" +88db01b2-8bcd-4671-9447-f466279f21f8,"" +890cdafa-a6ba-47df-9cc6-5ea4d4febcfe,"" +893f9661-bc71-4f65-83de-6483e73bbce3,"" +8979e8fc-9f34-4305-80fe-122fc619c7dd,"" +89cefdfd-b6d9-41fa-ab28-50563fe44a74,"" +89da332a-d11f-4f77-977f-6275f337495c,true +89ee0500-71a0-47c2-a59a-1b681d9a92fb,"" +8a31ea5f-804f-4b8b-8f91-6ac4bea2c4dd,true +8a56157c-ea64-4f80-81db-0c89a9d0b547,"" +8a5eacbe-4ee5-4e3f-9a0d-d0d1a13ae2ed,"" +8a8432ce-47e9-4fff-b71a-3147ad298fd0,"" +8ab69964-40a4-4d5e-8d2d-82d07174989c,"" +8ad34764-84db-4613-8483-4e8281f8f984,true +8bbcf675-5176-4df5-bbeb-38603e8d9c42,"" +8bf02dec-bd39-4a1d-b074-e0ef2167b93a,"" +8bfb6668-fc94-4b2d-9ec2-3385f1780d58,"" +8c39345a-45b8-44db-b5f3-6b5914b3fd93,"" +8c492ef6-59bb-4ccd-bc64-48d58d3da475,true +8cda568d-3e48-4e77-afa1-374809ed3c3a,"" +8d671817-97a1-46c9-b23b-e3013f13edb4,"" +8d98567b-2930-47f8-ad39-759745179c38,"" +8dcdbdec-4dd9-4928-9e46-4f46fd6df9dc,"" +8e1303b5-0694-4bc3-a0f1-559d51e05364,"" +8e55e68c-f1ca-468b-b4f7-11c11c2956cd,"" +8e5ebd37-bb67-4c84-b9ec-3422ed7c3b19,true +8e9c8ff0-82b8-4974-91a7-888dce4f7f8d,"" +8ea94f05-b5b6-4aeb-987a-567f627a4216,"" +8ef5e898-02ca-4617-b966-d5954b2cf403,"" +8f051932-7f43-4faa-b3b4-c2254756c603,true +8f19178b-b9b8-4858-8f94-c0f1e6eaf2c9,"" +8f5f16b4-fcef-4a03-98c8-a6aecea249f0,"" +8f62568e-7778-4729-a454-369f5a3f12f9,"" +8f62ada0-3878-437b-aeca-d59d658ece1e,"" +8fc9a4e9-9ca1-4144-8cf3-032a5ed6bac2,"" +8fea3b61-246a-4333-ad04-78348cb59585,"" +8ff0c7a4-f1c3-400b-bb31-c66df4866cfc,"" +900405a8-53f2-4a6a-bb75-80e9725f3439,"" +90126aaa-1dde-4136-9be1-4e30e2c1b667,"" +90e6ec4e-bc73-413b-a07a-e8d3b675c027,"" +91357869-20f4-4e58-8237-5b028e1f733e,"" +915eb7b9-47d0-4abd-a424-29605ef5f497,"" +91874305-9f9f-44b1-bac3-28dc0fabd0d2,"" +91f7217e-5c7c-4462-9f7a-ad7234bddf6d,"" +929edb45-2558-4fe7-b87b-ad4a801a26eb,true +92e25da6-dacd-4d0c-9457-751bc135c126,true +9307ddb3-12ef-414f-beb2-62a145d3f63f,"" +933302fd-1717-4cef-852f-0bc28dcfea35,"" +937de252-6da2-4dae-a80c-8ef39c6eb99b,true +93851c17-f570-4055-b6bf-ff928624666d,"" +9389f1c7-bba0-4286-ab0e-73ca1430824b,"" +9391e021-6734-4c01-a6d4-a9e151e58201,true +93c74a74-55b1-4f45-8b7f-3a0442ea5006,"" +93f749e0-9ee0-4cfc-b970-20f1757aa207,"" +94b95bc4-3355-4881-93ed-5db3dbcabc26,"" +94c1cc51-4dc2-4dd2-8a48-b6793675a44c,"" +95accbba-6ae9-4e70-9984-235e7371234c,"" +95ce9659-ea2b-43a9-a6cb-0b85fb4d9a48,"" +95e68649-5482-4d3b-bc6d-ca77d83531a6,true +95f21d98-e820-4054-970e-ab04c6b0ef69,"" +960866ac-7fbe-4f4c-8a41-f5c6d756a394,"" +962aaceb-7115-481d-8496-5ee5d12ac43c,true +96526997-d779-49ea-999c-43bd07d250e0,"" +96a5c8b4-5c9d-4092-9171-03da2422df80,"" +96df629e-35be-42c9-ad60-2fe43dfba0f1,true +975a8040-5330-47c9-9bb6-13a1d07fd3c9,"" +979229e7-5022-4a8e-9f23-8673564bbf4e,"" +97bd3c96-d38e-4a46-9e9e-0ff3466b80a6,true +97d8b079-ffd3-4bfe-a1a4-da06038813eb,"" +97f558d6-5624-426b-b21a-6a92c463d004,"" +98216728-4494-43ee-9af2-930201bcf2fc,"" +9873061d-c9e3-48f8-9a27-6a92255869c6,"" +98da112b-cca4-4bfb-9b85-32accfe1d295,"" +98e90113-a97a-40fb-b6e3-dd5f2e646901,true +993d7bc1-e96f-477f-b7e4-ad8d25c59e94,"" +995d662c-8641-4089-bb52-417a2eaba272,true +996a443e-1ca6-4e05-b1d8-b84455f2412c,"" +99893b11-009f-4ae2-9ce4-c0db476800bb,"" +99929bed-454e-4131-8368-9d152ab1da8a,"" +99b31b91-ec68-4c94-b23f-2b75d9302cea,true +9a1f1ac4-3ad5-4ada-80b2-cc31176393a6,"" +9ab5ad8c-b789-482b-838b-6cdcbd184330,"" +9acec191-7d26-4519-aac1-b50e00f3a2db,"" +9ad11a61-23e0-4603-90fe-c3001fdc7320,"" +9b4a1f69-52d8-4c2a-9e8c-072f71c2bd18,"" +9b876e7b-aa09-4b8f-8c58-a0cab74580b3,true +9b9b76e9-88d4-4440-868a-c5b153c277d9,true +9be9764e-cac0-489a-85e7-9f3c995e7061,"" +9c3ab93c-6424-4402-bd4f-0f0f7dd2b265,"" +9c630f43-c0db-42b7-b223-607d5a9a39a5,"" +9c9cebe1-ea55-48c0-9edf-22033f5ebebd,"" +9cb03b1e-f448-4f94-8c42-7f9271202a1c,"" +9cc11154-43fe-4bbc-a9ee-c5b8e0158d51,"" +9cece101-e41b-4d77-be20-ae0c0cd2cb9f,"" +9d3c20e2-321f-488d-9472-5756fa061ca0,"" +9d415448-4c3d-40c3-b910-ee3ff0df9b3c,"" +9d495d7d-837a-4d3b-bb10-608f9abc919e,"" +9ddbf3cc-7d54-4532-a670-5c361df676d4,"" +9ddc0101-8632-4103-a12b-bf8c5590f624,true +9deb306d-e6ef-4bdb-9a20-c9f5ebc556d5,"" +9e1b3c33-ef02-48ce-9b60-15b62a9a499e,true +9e32e6c5-44d7-45b2-bb87-65e98f8464b0,true +9e3fc42d-d08c-4cb4-93a0-a567c5f6a19f,"" +9e5071df-d3ea-4bd0-83b7-e08ba3fbb896,"" +9e58de28-a944-4ca2-b53b-b8da836ebd93,"" +9ea9ede3-8f0d-43c0-9f35-cc9fcfb622e3,"" +9eac5165-a8e6-4acf-8d52-4ad49e2ba851,true +9ed868fd-baf3-490d-b665-113b9427520d,"" +9f078010-f724-422a-8d8c-8855f2513eec,"" +9f3630e5-1b35-43bb-b53f-c191634ba14b,"" +9fa9a6d4-e6fe-4eba-bbe4-7319b735c63b,"" +a0238799-d0f0-49a9-96b4-33cfc1688279,"" +a0520e0d-7dc2-4218-9176-32eb842de85f,"" +a05480c6-8b4a-4406-9226-8dce4bbb4d21,"" +a0eff999-b331-46a2-b8dc-16727efc7580,"" +a0f58ee1-6e61-41a9-b0c7-5acdb005ac0d,"" +a1007ed7-d907-443d-813a-0bbb0ed84e50,"" +a1155987-73aa-4d48-b127-5da6d4cb7b02,"" +a1bd990a-7d22-47af-8258-5d4a05b269a6,"" +a1e0979c-d57c-4482-9087-88077cf657f9,true +a2224a99-ae98-4018-8651-38f1f145c76b,true +a2466ef3-db84-4434-9e07-af11c15c7490,"" +a253f260-5b6f-40a2-b4cc-d2c66856ffcb,"" +a27b5fe8-3465-4b6f-b63d-cca66626727f,"" +a28cbadf-918a-4831-be0e-9ab6c83133d7,"" +a2a6355a-2c94-4da3-a707-cc6db6553ce2,"" +a2bf4c9c-46d8-41c4-8eb3-044e832c78c6,true +a2cb445a-10da-4b47-add1-16584c34d8e3,"" +a3240817-7453-496f-9d7b-058889858399,"" +a34042e1-cc97-476e-8564-382d941458b4,"" +a35cdf18-7bae-4776-8aaa-652bfc135af9,"" +a3b8ad3e-c982-4023-8977-e9717a33d645,"" +a3f4de14-3398-4b34-9165-3a24c5742de1,"" +a3f9fe69-1527-4d04-a51d-ca9f1955d210,"" +a452f6e1-013f-4745-aa25-e2cd16ca4423,true +a46ec80d-9cd6-49d6-863c-5f5425899445,"" +a49f38eb-8de7-49e1-b7de-97ab32fb277e,"" +a4b245ca-a50f-4bb2-ad64-1124c31eb166,"" +a4d85c0a-4687-4454-89be-085b9a2632f9,true +a4fec49a-dc46-496d-86a7-5387c2490c3b,true +a500b66b-fab5-4ee9-bda7-11fb6170fff0,"" +a533451e-9e8e-448e-bc5c-1beace2c397c,true +a5a464c0-3b64-4ccc-bdb4-62aac198eddc,"" +a5c68c36-129c-482e-a9eb-023d7ec1ef0d,"" +a5d76237-0ea6-4016-91a1-33463ef1e9e4,"" +a60eddcd-ef19-4a84-ac8f-9206ed2aeed0,"" +a618792b-5b12-4545-a3e7-c07313d188f3,"" +a65e37e5-cf18-488e-8870-e4aeb021ee41,"" +a674a795-2553-478c-a5e4-6fbde13d7523,"" +a67aa655-62ae-4905-8042-30fd3ea68944,true +a6a1fe2b-4322-4ea8-9ca9-3bcf288afe36,"" +a6d86605-7084-4477-8c8a-8cf25c4d1b9b,true +a6fbb09b-5b17-49f7-a510-5c1596b424a3,"" +a717a4a4-b0b2-4aba-af9d-7b431e7883e2,true +a71de636-d8aa-4455-ac06-932ae6d848ee,true +a7575150-4148-4b9c-b941-64db8cc61356,"" +a769b6ce-379a-4f6d-ad06-5df3a58d25ff,"" +a7a4ad54-167b-4cc8-8f88-dcb555c9dfb4,"" +a7b38940-3cd3-43b1-bba9-6adb6090d1d3,"" +a82170ad-dc53-4899-a7ee-cdbb50e6a91e,"" +a83fed45-f814-46b9-9fdb-50892a1a1736,true +a8624bea-ae5f-4fd8-b6ed-11b0fbba9254,"" +a863d462-6836-4f42-a4a7-8925574b76fd,"" +a8c9520b-bfa8-4915-bd66-a6133e1c5ffe,"" +a8f862f6-2e8b-4cec-81ab-eb3be51d71a8,"" +a906f033-5f9b-4c1e-aaff-287cffc6cc7f,true +a93d01a3-5bc6-40c2-9895-64a21fbd6253,true +a9584e1e-deb8-45f9-830c-bef1caeb21c8,"" +a961ce1f-cf0e-42f7-ada6-f283daeac650,true +a972425d-8c4b-48ab-ae2e-fc7db6661f67,"" +a9888a76-f2fd-4161-9d15-6126ba398899,"" +a9b103bc-f890-4d91-9714-8d674aa842fa,"" +aa5fe96b-324a-4010-b128-a5063ce0be77,"" +aa86bd26-a862-4a03-9d8c-a04ec4c70c0e,"" +aaada565-8a59-4f69-b92a-a6cd9a323a24,true +ab3e98bd-050f-4bfa-b6e9-d829291290a0,true +ab50086c-5cac-4c6b-8e71-6988453228aa,"" +ab52bac3-d430-4700-b1e6-7efe2f5613da,true +ab63c859-8b0c-4874-9404-cf7c89a368e8,"" +ab8aa7ee-f65c-4a29-9995-ef33c4135035,"" +abcfc7ff-d824-4414-a586-3c33c263a990,"" +ac139e4b-9d46-460e-92d6-0ac43ce8cb67,"" +ac5bafbf-2bfe-4dc4-8e13-adb2d80e8dee,"" +ac645c40-0c51-4d77-a5fd-32e619019c60,"" +accb0b6f-b012-42cd-9374-91355ed49aa8,"" +acd3b2a0-1a81-4738-9bbc-9ab0c56b0a85,"" +acd69c03-a3c4-44fa-b3d6-749476309db9,true +ace8301d-b547-4cb2-8311-e27a8980633a,true +ad07d661-6987-4928-93a0-e7b902b24ffd,"" +ad2a1d8e-6e19-4d5b-90a7-ab24050735a2,"" +ad48d6a7-890b-4eba-919c-ff6b63e35304,"" +adad08de-aeef-42cc-9a2d-b99fab5ab1dd,"" +adbb15f3-b71e-49a7-a954-2f117a93b972,"" +ade38b82-c9cd-4a9c-a7f6-23130baac87e,true +ae13da32-7194-4f04-9fd5-c7f114a2e5b0,true +ae4da701-069d-4ede-b59f-a2524c6529f0,"" +ae52de8a-deeb-4144-bfd3-52c2cab5fbb0,"" +ae56f2bc-f7f6-4bc3-ad8a-3b1193e40f71,"" +aea61272-bb6b-4fab-9817-584a8fa2f55b,"" +aec815b1-23cf-4504-8576-fe03b8798092,"" +aee86188-c252-4910-98cf-a8d3ca9d029a,true +aef2f04d-7449-4088-9ab5-a227d1d7c36d,"" +aef45e8a-6923-4db5-9bb0-18f472f7f0ee,"" +af12db23-5b03-4e68-9127-acc14775690a,true +af52f728-fcd9-464a-9e25-28013b84fa1f,"" +af73e4bf-2ba0-44ec-b619-a04f1c13c9a1,true +af77032e-b1e1-42c5-8540-68bcb906f134,true +afd2cfb1-57b0-44ab-a8f5-6edc92bc870b,"" +b0138fc8-736a-433d-831b-0bc39281d67d,"" +b0525467-772d-4d5f-b326-095fe72890e9,"" +b07bd4e7-6586-49d4-9f43-19d58889743f,"" +b0a1b45a-9dfb-482b-96d4-362cd1ee625a,"" +b0a510f8-2c98-436d-841a-eee086eba241,"" +b0ad5c48-9538-493c-b5a7-87f8f5ff858a,"" +b0b8b0c1-8184-401e-9346-37422a6f2f81,"" +b0d0e0b3-acc5-4887-a39e-df765111f99c,"" +b0e1d515-5be7-409a-8c86-3aab8ab6787a,"" +b101ebeb-ddbc-4bb2-b5c9-2bc6a23e0a16,"" +b109fc82-447e-4d4a-a447-1c7f0569bbc9,"" +b12f7f41-f8f8-4eb6-b2fa-dd5a18639c11,"" +b155b9d3-54b8-4906-a564-601bf123f027,true +b18674a0-77ec-4d7a-b563-aad0856ce45e,"" +b2071f60-d582-424d-b873-9ea4c2615442,"" +b217ba96-19d5-4184-a1ce-890bfdec478a,"" +b243e613-8498-427d-bc20-acc54786ed44,"" +b2503fbd-507b-44db-b4d0-32700a3ba5f5,"" +b260cc9d-644b-4e25-a99e-9afc44dc3879,true +b2766451-0904-4ccb-90c3-fa7fcdbe19c4,true +b2b2289a-0762-4846-a4a9-fbd7dc1a3822,true +b2f8bfb5-d8a4-4df9-8724-bbce63e4dcdb,"" +b3394219-debb-4c75-988e-98fef99f0500,"" +b34ef85b-d2fd-4fbd-bb2f-6c21d737e21a,"" +b376a344-8079-4b95-99a9-d515f9dfa4a3,"" +b379efa9-29ce-44ad-b653-9cc03ac2bfe6,true +b3861c55-ae00-4319-8a76-03627b8c8bf4,true +b42a782d-79e6-4276-9fa8-744d5d1bea24,true +b444d377-d1b0-4eb3-aca4-c24d3fac8c37,"" +b4771fd0-758c-413d-bbc2-17e8e2cc79bd,"" +b48e7fa4-1922-4d4b-858f-a4f3bd8810a0,"" +b4a9bcfc-73ae-4798-a221-bbfe9991e0f0,"" +b4ce60ee-7591-48eb-81ec-0466d931b2a1,"" +b4f80b37-afaa-4974-9cd7-f53d788b2be0,"" +b508dc50-ca89-4b92-a12b-b032d3bf3a5f,"" +b516b7eb-4029-454b-a8ec-9bdede40ffb7,"" +b55a959a-73e0-4b7c-bb03-88bb22844c09,true +b59c5c8e-7d5f-48bf-b3d8-900e18a64b12,true +b67c717e-f9fc-4ec3-8cdc-ade1c8d1b95f,"" +b6d48a75-7d74-45e6-9fb0-154ca5264744,true +b6ed4e6f-6911-4cd5-92de-f50e8d1916e4,"" +b7289502-ed78-469e-8b73-04811493a484,"" +b7597773-8d3f-4ee7-b54a-6c8b60b6d47a,"" +b762619c-c328-4367-99a9-41f10751cedf,"" +b7d87d24-1805-4a12-8f8d-2d14509d5eab,"" +b7f806c7-6ed8-4b7c-bf42-dae336e3f7b3,"" +b80e31af-6b74-4685-a83e-50c5d7f762fe,"" +b862117b-c899-4938-9df3-6baf43911d1b,"" +b89dd640-c21c-413c-8774-f9a154d897f8,"" +b9578448-2338-4f49-9d63-5b886a537f07,"" +b9701217-a4ec-4ef4-84f8-49ae5825b45f,true +b97f2f71-17ed-4e3b-8e3c-374241e6faa6,"" +b982af8c-9e7b-4dba-ba74-76770efffefe,"" +b98be876-6fcc-41e8-ad4f-a78b1661acb1,"" +b9dc5e38-47af-4d07-8ad1-b0425c5cb386,true +ba047f08-841c-4d5c-8f98-b3e614906d17,true +ba5a0e2c-6e96-44d3-800a-991a81337994,"" +ba7bf8c1-e6a4-4437-b1ef-cf9e443971d0,"" +babdc581-4c93-4674-8743-1be4b18ba156,"" +bb182025-e149-489f-b69f-debf6907da09,true +bb2c53fc-6d97-4747-a60b-8010f62b7f63,"" +bb770b6e-11d2-457f-858e-2e2c6c86c0bf,"" +bb94d817-e843-416a-bc31-4858001e7976,"" +bb9c95de-6073-4239-bb5d-023d18344157,"" +bbcd2900-31df-492c-9d43-e8e0243f7e1b,"" +bbe0b9ec-5a6a-44a9-9770-fbe4117593da,"" +bbf5580d-bef6-4cd7-a231-b6a532d68aed,true +bc1bab6f-5d84-4c95-afa7-b4ab39038ced,"" +bc23a56d-f42d-4bdd-b704-07376d4a24af,"" +bc6abddc-4fa4-4ae8-97f4-9c566558b227,true +bc75ec01-01c4-463d-b64b-597c106d4eac,"" +bc929bf3-06ac-408b-a17f-6ecdb28759a7,true +bca14431-d108-4ceb-831b-b6cedb7b1b22,true +bca1f5dd-414a-4570-9f0f-ed733bd170f6,"" +bcec2bc4-4e04-48cf-b52c-8e1fa2ac3875,"" +bcf5154c-180c-4366-836d-13aa6bfce980,"" +bd3b7896-2b3e-4c1a-a2b0-aca1c2ea23c1,"" +bd687828-857d-440d-836e-5230d3254b1c,true +bddfe37f-fb3f-4a24-9ce9-181f250aa0b8,"" +bde29534-91ef-4817-a1f0-00a5cc9475bd,true +be196f96-4e26-4c23-8241-b0a10a39e831,"" +be1ca606-c535-4751-b52e-c61cfe9b6c73,"" +be3057cb-9635-4a25-863f-b4e79cda75c3,"" +be6d80a5-e2df-4e99-9721-80097129bd8c,true +be7a093e-d70a-4a23-9db3-bd95aca334a6,"" +beee46de-ddd6-4fe4-80bc-6f1f4b8fd42a,true +bef4846a-ca76-4000-a98a-00a9d327107d,"" +bf88c593-87b8-46d5-808e-8a332c5b50d5,true +c015fb2e-4062-417d-86be-681d98f8a13b,true +c0381889-4401-4bc4-9056-641340fa7ec9,"" +c070e6de-9ae3-42fc-9cf3-023839ad0442,true +c086a87e-ec86-403c-8bc6-37a9b77924f0,true +c095cfbf-cd84-4681-ba67-7b3ed318186b,true +c0a2c5cd-58d4-49ca-baa7-d5859abc680b,"" +c1003a14-56be-43d0-9aed-41fb6b048273,"" +c1370f98-3833-49d3-b58a-21c581ac4ab2,"" +c14127c7-9943-471c-b5c8-ee9f0bb95bd7,"" +c1d430d5-32b8-4410-a1b5-6212b2c411ec,"" +c2253c94-aa7a-4e88-8410-e528b08654f2,"" +c24b807b-174b-4e5f-91d2-93599fb21548,true +c25d3d58-1f6d-4004-85c4-3a162dbf48ec,"" +c286cd77-3c89-446c-a91a-4769aa088be1,"" +c297cce1-5d10-48da-8ef9-e860ac55f387,"" +c2a32eca-f03c-4fe3-8c29-3b69c9798b5a,true +c34d92cb-7086-40e9-b303-97e204d2ab4a,true +c36767f9-a3d8-4f57-a370-4a088ea0b023,"" +c37a2b82-a495-4d05-8f8b-99a757d0b83d,"" +c391f109-1259-47f6-979b-e72492709b66,"" +c3aaa4df-ead6-4a84-982d-3c28752e1070,true +c3ab4714-5eda-433e-b461-c1710b769476,"" +c3de8ca9-c03a-4689-8701-dd524884b5b5,"" +c3e98495-15f0-4040-b92c-31aa40235dcf,"" +c47474bb-570e-45af-ad38-8f5352ee3774,"" +c4757dc4-d69a-4e2b-9b8d-61353a5b21e7,"" +c4e0ccb4-df43-4b20-873d-3a66faab42b1,"" +c4e31ea7-6ad0-42b7-8579-c3b7fcafa740,"" +c4e6e6bd-2135-47e8-81ae-7e1d6d5e7482,"" +c4f82d4f-f53a-4d32-bc78-67aa7786ed1e,true +c52dbae3-07a4-4039-b04e-d79a86941c76,true +c52de404-7bbd-4b07-95a0-a66df22a3221,"" +c533b9cd-e56b-4e32-93b2-1189f37eb646,true +c557b460-3e1e-4ec4-9b4d-ea7bc0c8d64c,"" +c5609af5-9f49-49f1-80ac-b18fccbc9720,true +c5f20b88-cbd1-4283-ad46-5b0bf31fa3ba,"" +c6386600-4824-43b9-b366-4504f4e74d66,"" +c6442145-3aad-4b57-b2cb-b7eff0699959,"" +c6700fda-f009-496e-83ec-0532f34bfa92,"" +c6a0f008-4d97-4319-8099-7e0cb6708438,true +c6a3be9a-b061-4a5f-879c-e948aa895f3a,"" +c6b44473-ebbc-48dc-8a42-44b7da0de840,"" +c6cb9c06-0c13-4cd7-8d20-ca7b99f7d141,"" +c761f0d9-c918-41b3-9131-856aad1f5b2b,"" +c7e313a6-4659-4fad-9402-946879a91616,"" +c7f263ed-22de-479e-909b-f6ceaf5f2eb0,"" +c80ab964-cf3d-4e89-9232-8e787cb0def5,"" +c80ec189-5f9e-4049-922e-969d28b3bdde,"" +c837974e-0093-438d-a822-19e97249f0dd,true +c895019e-6471-423d-ba15-4f06ed628c7f,"" +c8bbf035-8a01-4c57-934e-988a584c98b9,"" +c8c936bd-3e6e-4500-bbd5-051659c59309,"" +c8cb1ae0-6a31-4869-a396-78dd5dc7f302,"" +c8f123b5-8e57-4c9a-8770-bd4b4c2c064a,"" +c8fd40e1-35fe-4c8b-af6b-730fd6fb5fab,true +c92a4616-7215-4ba1-99df-40aea82eedc3,"" +c9445880-656a-4def-ad1b-2a4200d76c13,"" +c9647fa4-976a-4d9f-9be4-26b9c71f6eb1,"" +c98372ea-e9ae-4438-8715-b54ccdcc8f30,"" +c99c2510-b392-42ee-97e1-16f6a3b50978,"" +c9cd4daa-907f-490d-b3e5-a43eedd74c32,"" +ca45fe64-e7c9-48ea-bc02-3bc6a0b58e46,"" +cab21bd5-4314-4bb1-8b16-0967f8b08757,"" +cab53124-f25d-4b4a-a238-3d028e788b2e,"" +cac06963-2da1-4935-a6c3-d6152132c7eb,"" +cad24afe-7131-4158-9153-742c9fd16c9b,"" +cb1c6a1f-e659-4fe2-9cfd-f28e776a4f06,"" +cb5c302b-35a6-4d8d-8d6f-72ad1a3ed1cd,"" +cba58256-d558-48df-83d7-be5939d6dfab,"" +cbafa157-0905-418d-983e-827cc88faa94,true +cbfd98df-8ac2-4497-988b-bc4d1c9c6095,"" +cc0b8f72-76dd-4eaa-84c1-c343982e18e2,"" +cc1e584e-6752-4bf3-8c51-ac0192b5df44,"" +cc372f1d-a83f-438e-8d8e-c346eb03fee6,"" +cc38b529-3137-4d89-804e-51c6bd60ca97,true +cc3a3e25-87b0-4012-869b-9905b31f8319,true +cc5fd0ac-6ce5-43a6-bb4a-67279156ee8c,true +cc7e521d-4fa6-4215-942a-48f84ddf9a49,"" +cc8ade38-d9f1-4700-aad3-dc866f35d1a1,"" +cca1c9e8-c68f-4fe0-ae79-693399d8ee31,true +ccb56a74-40db-4d58-b0c4-cd485e8a7fe8,"" +ccb810a0-28a5-4248-92dd-7259a0fa0c22,"" +ccbda4c9-3ec5-46c1-a8d6-a215651dfb43,"" +ccbda933-c3d4-44bc-acfc-fd26bc234bb6,true +ccd5afad-8f2f-4305-b646-26137eff0a0d,"" +ccd69a01-cd72-41e7-87b9-de8454411ee3,true +cce63ba1-abd2-45d1-9ba0-a2cd0fa9b00e,"" +cce6dbc6-3a49-4f8c-b908-e6f11be81ac6,"" +ccf36cc5-5009-4fbc-8e8e-81f38d3195a7,"" +cd013d11-8609-41d9-a9c6-7fb33a499f51,"" +cd09f5d3-c846-41e7-a1ff-aabb55973663,true +cd2c3f78-ddb1-45f7-9939-987e070d9414,true +cd3d18de-96d3-4132-ae73-b80795a53ad0,"" +cd71ffde-9d7e-4c27-bc22-306f81cb46b7,"" +cdc96f77-a589-4ae4-85b6-c6d2c89487ba,"" +cdcad0ef-8049-41c2-a8ee-21835323b05d,true +cdcae470-3531-4548-a2a9-ba665fca8930,"" +cdfbf7e0-ad10-4c22-9ee4-888bb6bdd198,"" +ce1b7543-fee6-4267-a00d-daa5a5ea2861,"" +ce225925-36ec-4c1c-a327-c64a146bef46,"" +ce512c4f-3808-4ef6-9853-6f7d14e146f9,"" +ce7037fc-2614-46f4-a8db-40a2a0f9e3ac,true +cea2e600-da22-42b8-aa08-f0e951ff35d5,"" +ceea3ce9-5ab6-4cbe-99b2-34ca9e6eb468,true +cf17f19b-9f66-4391-bd2c-55384690d9bf,"" +cfced5ea-1e8e-4c85-8318-ef91a041f4e0,true +cfff2bee-7cf4-459f-8690-3638266a07e3,true +d000b381-5ba8-4a5b-84c7-f371e104eeaa,"" +d012051d-65bd-4a59-b4cd-f454d6010888,"" +d05fad91-9a3d-4d8b-ba99-5a33fcc82081,"" +d083a1aa-9662-482a-8192-4709f904f16f,"" +d0b22c1a-d0f6-4d83-bd3c-475a3acb0b83,"" +d0be6a2c-ebc5-405a-a2a5-006a5ea52c76,"" +d15e2267-a699-4f43-96a3-e98de7ace044,"" +d168f31c-483c-4340-a2f4-6a60cc8cab30,true +d17dfd97-60aa-4313-bcd7-f48935faa2e8,"" +d19385b9-bc04-4000-bddf-1d09425a2b4f,true +d1ba146a-9503-472e-b668-3686ff275ef2,"" +d1c2f4a3-8af5-4ad4-bd0c-032b947d3c25,"" +d21af0cb-c5a1-401d-b605-a2e3ba3f46a1,"" +d2272949-75dd-4949-88fc-f0f157c83105,"" +d230ce54-d5fe-4983-9e91-519ce1249322,"" +d241d156-478f-47a0-989a-96a5fff36780,"" +d2663716-cdce-4d88-90f8-51fb5d59aa4d,"" +d2a3d703-44e0-4dee-a8bf-9fc0457841ae,true +d2c11b5d-93c6-4500-bc4c-8dc50057f362,"" +d2c4f328-e7a4-4d87-b180-9a0eee13d165,"" +d2d08017-f955-4e0e-baf0-2d5d37acc304,"" +d31bee09-f02b-44c5-aa15-a06343e0bf9f,true +d355c55f-53b5-407e-8785-33e4e54bb3fe,"" +d37a07ee-5d7b-41d0-bbf1-331277662c52,"" +d397d6be-6008-47bb-9f72-628fb5c41cb1,true +d3997124-9e16-4df3-8c60-f43f62bbf036,true +d3e6de0a-bfef-438a-b982-4818145651b9,true +d3f2c0c5-1f4c-4156-8cc0-e2d87db500ea,true +d44741c1-d58a-495e-9f30-cc0f19fafc4f,"" +d46e0f9c-5dd1-4e88-9a22-96ca436674d3,"" +d4aa494b-3328-48fa-9873-39be6536a437,"" +d4dc8d84-20e2-4053-b244-47549e74680c,"" +d524b58b-90df-4d29-b4c8-308ac04dda2b,"" +d56569d3-0a16-4968-9470-93d410916f11,"" +d5944a91-3bf9-4915-8512-0f1f50a8eb7c,true +d5aa4b50-1ac3-48bc-90ee-7ad6bd78a078,true +d5ca4448-871c-4175-801f-38bb43cdb48d,"" +d60c26c8-1b83-4de9-be17-81fca1ef2575,"" +d6c622c5-3662-4b29-9180-0b4055afa612,"" +d6f1073c-bf1c-4232-8ee2-5b92f0a32d8e,"" +d702593f-4121-4ac3-bc3f-dae097c7a044,"" +d702f666-bb61-4d55-9579-e56efc3e0e91,true +d704d649-9e08-4502-a9c5-820f503dac9a,true +d7c0876e-3031-4be6-aa2b-bbd8e2bd8884,true +d7eba5f5-103c-4442-9a7f-62c81a9dd9e5,"" +d7f76663-5195-4de4-bc43-ee61e467fee4,"" +d8534558-7fea-4dcb-a1fb-2981b725dd61,"" +d86d32ef-f50e-46d6-b434-fd03bc807cc4,"" +d8a42c84-6930-4b12-8c1f-9a492d290454,"" +d8cdd01f-c8b2-46fc-8f69-35da565e1b99,"" +d909c299-c622-4c39-829f-415621e09eb1,"" +d94160a6-7943-496d-8b66-172e9ec6804e,"" +d97e5072-19d7-49bb-aaf1-f3a56816f3e3,"" +d9e59fa9-630b-4d97-aa71-4e03bd6fe1ad,"" +d9ebe2d3-2a6e-4079-8f59-43a327e68ebf,"" +d9fb7469-d50c-4492-8c1c-8bac1ccf81bd,"" +da33cd96-8f15-47f1-aeaa-846234a6b9cd,true +da412e67-cea6-4ad9-8c66-75bf7bd9d0f9,true +da438390-6d67-444a-be2e-354a21d30559,"" +da4c7a8d-c75b-4050-bbac-5edb50e48d4e,true +da5de060-6e29-4379-9119-175b394f9bac,"" +da98f3f0-236f-4c41-b123-09f1fdb5bfc6,true +daa0c0a8-91bc-41c5-9627-adad04a7b15b,"" +dabbe6e8-3692-406f-8a2d-f81217eb6964,true +dac01aa1-44bc-4ff5-b123-4868f8f278ef,"" +dac87128-5796-449e-85ea-2c1f8e09ba0b,"" +dad4c0b8-0e02-470c-b7be-6477008d925f,true +dae5b7a2-ad95-411c-9721-067d3c578262,"" +dae62ac4-666c-48f7-8587-ff9330a9e9e6,"" +dafa1e16-2149-4e76-a693-88d2e7e29a0a,"" +db0a2a90-dc62-4e6e-9a46-88b9a3145ab8,"" +db16da7a-538a-4270-bd0e-1b4398a922a6,"" +db1788c0-f515-419f-ada6-834a37168001,"" +db17c5e1-6d1e-4b0b-a345-8dca770e7097,"" +db5a6739-388b-453c-99d0-58cfce24d5d5,"" +db8a9775-faff-49c4-8b35-74e92245f439,"" +dba78a31-21c5-4600-b18e-3e90237ab40b,"" +dbbc9b03-4d76-4118-8637-185659ce2adb,"" +dbc0cb40-5e33-468e-bc9e-2cf8ed602d01,true +dc2a8c49-ac89-487f-a22c-dc126a5c7ed1,"" +dc36732e-8054-4d6b-a860-e7c54f04a0bf,"" +dc567aee-5073-493e-bec1-b6b0f4e160d5,"" +dca15405-893d-4468-84ad-574e54f45c22,true +dca8db58-f563-4a61-b747-94f5ec5bf135,"" +dd09736a-85a6-4c2c-979b-f2a7027308d1,"" +dd9a2411-d45c-4385-a6cc-650df658ae9e,"" +ddccfc35-d71d-4e65-afd3-4cf2a21e607f,"" +ddda9bf7-d645-40a2-8aed-eb9efa425d01,"" +de1c574a-a49c-41d2-b321-849c005c89e0,"" +de81cdd8-a6ed-4ed7-af7b-6ee027d0d5cf,"" +de9bffc9-c6ee-4ad2-8055-1e0d0e48887d,true +dea4f32f-0f51-4bf0-af67-e79b34674fac,"" +dee72459-d0ba-4771-af17-f38f8396adb0,"" +deee1173-4f31-4784-8c63-d02e2a7b16ba,"" +df471b44-cfeb-43fe-9488-8f85a03952fa,"" +df529c86-f229-404c-8f0c-44e1b29e4e95,true +df54eb2c-59c6-46cf-9cc0-201e94acc07d,"" +df6522a8-552f-4ba1-906c-139f58a0a85e,true +df8a0f98-893b-400b-9da4-2514e50c0b24,true +dfa10a30-82d4-433b-9df5-b0d2291378ae,true +dfa72495-2737-4fe8-9fcb-c799ab51ddcb,"" +dfb67b9c-64b1-422f-887e-139a8f676fa4,"" +dfb6cb62-17e3-416b-9214-6ae6f432874a,true +dfef1286-acc9-44ee-a736-b9676812235b,"" +dffc6fbb-8592-4f0a-8190-1e64eb2bc634,true +e0010996-fa96-46be-9431-ebbe139c081c,"" +e00ec480-db7a-4aab-84ab-7e5e7184bfdc,true +e0ef5017-54ab-469b-9871-f706d4899465,"" +e0f2d024-4210-4f18-82ce-51755627e13f,"" +e1309aae-d59f-44f0-b630-fc538bab9027,true +e134ed0a-49e0-4a0d-b09b-4e874f422e60,true +e16400e0-a6e6-459a-a16d-361a7b2ffd1d,"" +e1a9f42d-103d-4e01-b98a-e70f98aa1c00,true +e1c6ba82-6bc7-4f15-b71c-c669cb78bcc9,true +e1e4e2e8-1c7b-4c67-bc7a-3940e6670357,true +e1f9bb17-3719-45d6-b4bd-3be88f17e429,true +e1faaf81-1761-4a4f-b01d-c179db8474f3,"" +e20067e7-f2fb-472b-811a-3d28a08bf31d,"" +e2501541-dbeb-4e28-b4d8-598e963d6158,"" +e2b04908-769e-4916-bbc4-45ad7db85e02,true +e2eccb3c-0755-49c8-8e82-cc510481ed04,"" +e2fbc85e-dbc4-422e-9a11-0007d5c194ae,"" +e3784666-7f48-455f-a137-e6262a617ccf,"" +e3c2c69c-bbe5-4678-aa6c-3e5c7ff996f2,"" +e3c5df86-3d35-4dd8-a081-48632a3b67c8,"" +e3c94ca7-a487-468d-83f2-8068a592a774,"" +e3e4b2a1-6c40-4f42-a5b5-17b352209b40,true +e47d814f-5139-4339-a710-fe953efcfa74,"" +e484c47e-4da7-425b-a636-de512fcbc822,true +e50c31e2-5d52-4d45-9b8f-8ae3fdd26488,"" +e53c4c4c-976f-4cb9-95af-fb11c264cb72,"" +e53ec08c-9029-4b50-bc1b-a302878b17e6,"" +e562454d-7fbe-43a2-8046-bfc5b6ab043b,"" +e5803192-2273-43bf-8acc-945cc13af70f,true +e5c7eb1c-238e-4bf8-82bb-205c383f1517,true +e5d9ff02-57a4-4e05-b573-0dbc80e19958,"" +e5fa00b8-1779-467d-b881-757b6796d8b1,"" +e5ffe973-b166-4240-b5d5-17eaad746c4d,"" +e6e9da2c-7684-41fb-9123-def8a7cb4dba,true +e7098f3c-9dec-4aa9-9554-b3ad9a7d842a,true +e73f4c39-965c-47ca-adb5-0544e20d45bc,"" +e757ee7c-a50b-4575-b383-587e6f782bed,"" +e780a3b4-6b8a-40a1-a9ba-5f95725399ea,"" +e78c789e-7b81-49ea-b7e7-d17d8e8925aa,"" +e79745a2-654a-46d6-a886-dd1d21acccf4,"" +e7b3ee1d-2d5e-4dd6-8ae0-ebddf36aee4d,"" +e7d53c39-4adb-441d-a4e1-50c5f60fd3ef,"" +e81ec213-328e-446b-9c33-ee80bc850d44,true +e83d90d6-c694-47fc-8120-e91546fdda8c,true +e8ba3683-aa88-4800-a248-737d39ecb6c6,true +e8f05409-642b-419f-8475-28449c5f9569,true +e952a654-0605-409f-ac45-19d00d4b53c5,"" +e97e9781-3cb3-44b0-89a9-b704d456fcc2,"" +e9ab227b-2a64-4c48-a8a0-6c166bf4b970,"" +ea2f0657-1625-414b-9c7c-bd0b01a12eff,true +ea91b2ad-37c8-4a4c-9178-11d7a5e4e75f,true +eac7851d-1d61-4a35-9f08-c5da53335f7f,"" +eace08fa-4018-40fa-a7fc-3daf10b2f21d,"" +eae386dd-ef2b-4629-817f-6f8c44ce5817,true +eb2419c3-3b91-4d73-b4be-a40e759ab957,"" +eb2781af-72a5-4760-a9a7-20d212f56230,"" +eb511148-31f2-4292-a83f-d16d3be6631e,"" +eba22453-8605-43d3-81f4-f703e58a230d,"" +ebad9165-a532-4846-aff7-5038cd5c694f,"" +ec3d7e48-c9f4-4f64-ada9-7e2891a62d1b,"" +ec462f5c-2aaf-45fa-8c25-6f556349af00,"" +ec4d4a14-0934-4d28-ad20-64db8fe78d42,true +ec9ccc2d-1e6d-4ebc-b1e5-86d493d19eb4,"" +ed352845-d1af-40f9-afca-53503a6518d8,true +ed7a65e1-de6f-4c2f-bd47-aa78d0c2189c,"" +edd41c4a-a5f3-4531-b506-d81c04e1ebc6,true +ee004b78-c05d-4c34-a9f3-290a9259d3a3,"" +ee6749cf-0622-4376-958c-229ebc0f9618,true +ee8905ec-84b9-4fa0-b1b5-0908206b7af9,true +eeaa2e25-739d-443e-ae83-b0ef04e2836f,"" +eed7cb9f-7fe4-4f9d-bb4d-cdaeea6e144f,"" +eeeed70d-3cc8-4b11-9f6e-ab44da414b90,true +ef079a39-9f59-4681-91fa-3d745c5eaec1,"" +ef10cf22-bdb9-44b5-bc9e-bb777ad8f090,"" +ef1c5dab-b3a9-444b-b825-8fe3d0248e02,true +ef9360c1-b9b4-4dd9-86de-f996cd22e7a1,"" +efbe6910-0629-46d6-bcfb-5880a1795742,"" +eff17621-813f-4e99-a307-118ac9273efc,"" +f018a615-d5ff-4c3e-84f8-d4ebb1f342bc,"" +f04f4b77-a9f2-4ce1-9d0d-6b43d39483f7,"" +f07f7d83-1fe3-4da4-9c33-86aae3ac67e5,"" +f08704a2-c1f5-44d5-a3f6-43c5608dc987,"" +f0a81ab6-10ea-4859-a064-b3b0dc662f24,"" +f177f59d-ecf0-41a6-b81b-a311eb760976,"" +f1a1aa8c-3779-4c42-8302-6006dcb95151,"" +f1b36b35-1bd3-4328-aea9-4c5b3a7c7404,true +f226c31e-37f4-4a4a-aa1f-aa400447a799,"" +f22fe03d-e1e9-40d5-9ff7-a5bb6e7a2d71,true +f27187d5-f8f0-4188-ae0c-f77ca494a9b3,"" +f27a59e9-c785-47d7-88a4-de7706bb4f92,"" +f28962fb-0214-4664-b9c3-7c67d4eb2e8d,true +f29f4e3c-81f9-49b1-bc52-47a60a3076bb,"" +f2fd6221-529b-42ca-a060-30db0996a398,true +f3047ad3-3524-4a48-a00f-603a65fa0891,"" +f310aaca-1c10-41fc-ac49-8430c4c23656,true +f314d140-929d-4ced-bbd9-c7f0204e99eb,"" +f32094e2-ab74-4567-98c7-e35fe529393a,"" +f3338731-d7b9-4c1f-8e3d-e73b5b756c7b,"" +f3deb81d-79ca-4176-9b6d-bd96bb5b7a16,"" +f3e99f4d-5bbb-4f11-9519-4d3aa15e25cb,true +f3fd705f-3f03-4f26-a477-f36e65fe7499,"" +f4077b1d-84fd-4489-b8b1-bbc9f5b61da2,"" +f475b875-3df0-49fc-b4f9-10b2ac1c40a6,"" +f4bba159-e786-4f15-8100-6bff75c03635,"" +f4e59f5d-12a4-4005-bdae-e157e145c119,"" +f5042323-b136-4c9d-bf57-0b7b70609495,"" +f51f7f2d-968a-4630-8e06-d6beb52b9f6b,true +f55f863e-ffbf-4eb5-8f76-e2069098adcf,"" +f568f305-55f4-4d1f-849f-6c8f371f5051,"" +f5dec406-3636-4ac4-894f-1035ef3ef570,"" +f5e51d8b-bda1-4093-b887-f2355ea171cd,true +f5ebecec-331b-4d57-b353-3f50538cae2a,"" +f5ed5cb8-adc0-424c-8371-ffb8e7cc9bc4,"" +f6514087-db2a-40d8-897a-21b5af7d590e,true +f65d9eb9-e3d4-42fd-8d3b-168a6009f6ed,true +f66efc52-e910-4d73-a9a7-fddc110c16c0,"" +f67e345d-7f27-48bf-afe4-00d79c3cf300,"" +f67f612c-6534-40d1-8e56-c056e886b98d,"" +f6b9b783-c8af-44c2-8443-44b80e972686,true +f6bc9618-f245-4402-b69e-74d424736c4b,true +f703fec9-ca54-4363-9ef0-4c4acdc37e0c,true +f73ca57b-a00f-4217-9106-aedd26812b3d,"" +f74957d5-eef8-4683-976f-c59ffabdeadc,true +f7d2872f-bfc1-496e-ae92-a9d446aa72da,true +f7d6ad2c-1f7e-47ef-adfc-16a5d4d0189a,"" +f8043ec1-8313-45f9-a6dc-4cc1e5ad5f54,true +f8673366-8964-4c6c-aa9f-02e0920f21dc,"" +f8683b38-42d3-4f32-a543-034c05dc07f6,"" +f8816dca-a41e-4ef9-8a17-b40fc7d707ad,"" +f89b5e47-c8c3-4067-99ea-bd22f8932ee1,"" +f8b6ebd6-8d2a-4592-ad16-06772ad32cc7,"" +f8ddb5b8-70ff-4d5b-adf0-1752811f7be2,"" +f8f99d6a-94dc-4fca-8772-d0c3fe887a7f,"" +f912a4cb-2eeb-41f3-a410-0ddc3eab9cdd,"" +f9185eb8-d2e5-47aa-916b-a5ff24612ddf,true +f92f93d5-3037-4692-a062-ee84b9961510,"" +f93aa334-4532-4774-b82c-6d040d3c7156,true +f93e975b-d07d-434d-b227-666e744892c7,"" +f95dafda-7f63-4ddb-acbf-e363dc901b1b,"" +f95f8351-818a-4cb1-b26a-507c4baced47,true +f9aff2d6-fe39-4d17-90cf-917edf5d2bc8,"" +f9f5480b-27da-44fd-84a6-1ee4d4a55e36,"" +fa0ee43e-abc9-44cb-9248-3acc6c40e693,"" +fa4daa52-528e-45f5-8dd6-fe2a2c9fe8ff,true +fa522b1a-3244-4203-966f-546dcfb0c62c,true +fa5e8aa4-84e3-44b7-9dfe-326abdf02162,"" +fa7087f0-0140-4c94-b7c9-185b89c613c8,"" +fa786b6e-dbf4-4c51-a8c6-3006a029e8ba,true +fa80db64-ef07-4b14-aa7d-e93585fd84e2,"" +faa80ca5-9c2a-40f4-aeec-26d982e663ae,"" +faeaaf02-2ae6-4f5b-b729-61fd56d3946c,"" +faf774dd-13ef-4b09-8cc9-3cf33a562482,true +fb176a2c-4b8f-40d7-bb81-f370c96c3f96,true +fb1df190-d719-42b7-8e9c-d7830f92882f,"" +fb42fd3a-123d-4b2b-8a2f-d7e37af0a675,"" +fb55aea6-a5e7-4a55-a8df-967f504023f6,true +fb93624f-3d94-4c5d-9aba-10a3eaea0434,"" +fba16e5e-1b98-484e-8296-43a3995b9f75,"" +fbacdef8-f2e5-4a1c-8a01-b25442a5af89,true +fc348304-8ef3-4b3d-afcc-a66718315d89,"" +fc6dd798-66ac-45e2-a663-ea69b976c120,true +fcdd2cbf-917f-4429-a41b-48f8fe2a188b,"" +fd325c37-4113-45e9-9e55-abd6b2ce6dbf,"" +fd694334-1bcc-40f0-863a-017311fbaecb,"" +fd75e2b2-c780-477c-a813-046ccec6fe51,"" +fd7a8bc2-9005-4230-a4f5-348f4b1f5b16,"" +fd7ef037-c576-498d-bdef-cec86fdd60b0,"" +fdaeb95c-9d66-4da9-87cf-eabbdc593007,"" +fdb161bd-ad0e-47fa-a5bf-f17ece98f28c,"" +fdb49a29-6648-4dc5-bf56-ef948b00d885,"" +fdbbdafe-ed7e-4bd1-ae67-86430153a738,"" +fdc10bfc-2b5b-4a23-a93a-eaa85803bbc3,"" +fdce38d2-ae3a-474f-8024-52279c43b745,"" +fdfcdb50-716b-4452-a4c8-e03d3b8f89a3,"" +fe4c4989-ff06-446e-bf01-5c5d8f69f1eb,"" +fe66bca4-eedf-42a9-8def-d29ef3dd01de,"" +fe719bdd-cc9a-4c93-b2c9-982d051fceb1,true +fe7906dc-a882-488e-b576-2ef3486bcf79,"" +fe7c2db1-c229-4681-9373-e388dccb9e37,"" +fe7cb896-d81d-4b38-885c-4134ea64020e,"" +fe97c34d-b787-4c3a-a972-821eb42a3442,true +fead13f8-4b89-4f33-950b-ee9254a36c4b,true +fead2d6e-2979-4de7-8118-698906954d06,true +ff9df3bf-b8fe-4789-aa34-b205b5143a92,true +ffab80f3-741c-4c67-af21-4585f1c05d2f,"" +ffb89ee2-c3d9-4fd5-b897-a0e0eb692143,"" +fff8bb82-5151-41f9-b150-801ed4957277,true diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index 73b85e7318..c6715b74e5 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -84,7 +84,7 @@ Examples: ```

- The implementation of Decimal within Pathling supports a precision of 26 and + The implementation of Decimal within Pathling supports a precision of 32 and a scale of 6.
From 843a7f3d04da338a5d3857fd1d419d66a93b1cc7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 4 May 2022 11:35:30 +1000 Subject: [PATCH 24/83] Cater for scenario where UCUM expression can't be parsed --- .../pathling/terminology/ucum/ComparableQuantity.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index 0b15b774ec..f09318199d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -22,6 +22,7 @@ import org.apache.spark.sql.types.DataType; import org.fhir.ucum.Decimal; import org.fhir.ucum.Pair; +import org.fhir.ucum.UcumException; import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; @@ -97,7 +98,13 @@ public Row call(@Nullable final Row row) throws Exception { // Use the UCUM library to get the canonical form of the Quantity. final int maxPrecision = DecimalPath.getDecimalType().precision(); final Decimal value = new Decimal(input.getValue().toPlainString(), maxPrecision); - @Nullable final Pair canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); + @Nullable final Pair canonical; + try { + canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); + } catch (final UcumException e) { + // If there is a problem parsing the UCUM expression, the Quantity is not comparable. + return null; + } // If the canonical form is not complete, we can't compare the quantities. if (canonical == null || canonical.getCode() == null || canonical.getValue() == null) { From b893856cdfc4af0076cb87cd3e665065b08fbad6 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 9 May 2022 16:53:08 +1000 Subject: [PATCH 25/83] Add Quantity canonicalization tests to encoders --- encoders/pom.xml | 11 ++- .../encoders/terminology/ucum/Ucum.java | 90 +++++++++++++++++++ .../encoders/LightweightFhirEncodersTest.java | 31 ++++++- .../encoders/SchemaConverterTest.java | 42 ++++++++- .../au/csiro/pathling/encoders/TestData.java | 13 +++ .../src/test}/resources/tx/ucum-essence.xml | 0 fhir-server/pom.xml | 11 +-- .../fhirpath/literal/QuantityLiteralPath.java | 2 +- .../fhirpath/parser/LiteralTermVisitor.java | 2 +- .../terminology/ucum/ComparableQuantity.java | 1 + .../csiro/pathling/terminology/ucum/Ucum.java | 37 -------- .../pathling/test/UnitTestDependencies.java | 2 +- 12 files changed, 188 insertions(+), 54 deletions(-) create mode 100644 encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java rename {fhir-server/src/main => encoders/src/test}/resources/tx/ucum-essence.xml (100%) delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java diff --git a/encoders/pom.xml b/encoders/pom.xml index 185783522b..4e1db646a5 100644 --- a/encoders/pom.xml +++ b/encoders/pom.xml @@ -13,8 +13,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 @@ -83,6 +83,13 @@ commons-compiler + + + org.fhir + ucum + 1.0.3 + + logback-classic diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java new file mode 100644 index 0000000000..af8c5949d0 --- /dev/null +++ b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java @@ -0,0 +1,90 @@ +/* + * This is a modified version of the Bunsen library, originally published at + * https://github.com/cerner/bunsen. + * + * Bunsen is copyright 2017 Cerner Innovation, Inc., and is licensed under + * the Apache License, version 2.0 (http://www.apache.org/licenses/LICENSE-2.0). + * + * These modifications are copyright © 2018-2022, Commonwealth Scientific + * and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. Licensed + * under the CSIRO Open Source Software Licence Agreement. + * + */ + +package au.csiro.pathling.encoders.terminology.ucum; + +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import java.io.InputStream; +import java.math.BigDecimal; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.fhir.ucum.Decimal; +import org.fhir.ucum.Pair; +import org.fhir.ucum.UcumEssenceService; +import org.fhir.ucum.UcumException; +import org.fhir.ucum.UcumService; + +/** + * Makes UCUM services available to the rest of the application. + * + * @author John Grimes + */ +public class Ucum { + + public static final String SYSTEM_URI = "http://unitsofmeasure.org"; + + private static final UcumService service; + + static { + final InputStream essenceStream = Ucum.class.getClassLoader() + .getResourceAsStream("tx/ucum-essence.xml"); + try { + service = new UcumEssenceService(essenceStream); + } catch (final UcumException e) { + throw new RuntimeException(e); + } + } + + @Nonnull + public static UcumService service() throws UcumException { + return service; + } + + @Nullable + public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, + @Nonnull final String code) { + try { + final Pair result = getCanonicalForm(value, code); + if (result == null) { + return null; + } + return new BigDecimal(result.getValue().asDecimal()); + } catch (final UcumException e) { + throw new RuntimeException(e); + } + } + + @Nullable + public static String getCanonicalCode(@Nonnull final BigDecimal value, + @Nonnull final String code) { + try { + final Pair result = getCanonicalForm(value, code); + if (result == null) { + return null; + } + return result.getCode(); + } catch (final UcumException e) { + throw new RuntimeException(e); + } + } + + @Nullable + private static Pair getCanonicalForm(final @Nonnull BigDecimal value, final @Nonnull String code) + throws UcumException { + final int maxPrecision = DecimalCustomCoder.decimalType().precision(); + final Decimal decimalValue = new Decimal(value.toPlainString(), maxPrecision); + @Nullable final Pair result = service.getCanonicalForm(new Pair(decimalValue, code)); + return result; + } + +} diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index 2582da7add..9ee0db774e 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -30,8 +30,13 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; import org.apache.spark.sql.catalyst.encoders.RowEncoder; -import org.hl7.fhir.r4.model.*; +import org.hl7.fhir.r4.model.BaseResource; +import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.MolecularSequence; import org.hl7.fhir.r4.model.MolecularSequence.MolecularSequenceQualityRocComponent; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.PlanDefinition; import org.hl7.fhir.r4.model.PlanDefinition.PlanDefinitionActionComponent; import org.json4s.jackson.JsonMethods; import org.junit.Test; @@ -201,4 +206,28 @@ public void testEncodesExtensions() { assertNotNull(stageTypeExtensions); assertStringExtension("uuid:ext12", "ext12", (Row) stageTypeExtensions.apply(0)); } + + @Test + public void testQuantityCanonicalization() { + final ExpressionEncoder encoder = fhirEncoders.of(Observation.class); + final Observation observation = TestData.newUcumObservation(); + + final ExpressionEncoder resolvedEncoder = EncoderUtils.defaultResolveAndBind( + encoder); + final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(observation); + + final ExpressionEncoder rowEncoder = EncoderUtils.defaultResolveAndBind( + RowEncoder.apply(encoder.schema())); + final Row observationRow = rowEncoder.createDeserializer().apply(serializedRow); + + final Row quantityRow = observationRow.getStruct(observationRow.fieldIndex("valueQuantity")); + final BigDecimal canonicalizedValue = quantityRow.getDecimal( + quantityRow.fieldIndex("canonicalized_value")); + final String canonicalizedCode = quantityRow.getString( + quantityRow.fieldIndex("canonicalized_code")); + + assertEquals(new BigDecimal("1.0"), canonicalizedValue); + assertEquals("kg", canonicalizedCode); + } + } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java index 31f53eb3dd..5126bbc543 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java @@ -25,12 +25,22 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.spark.sql.types.ArrayType; import org.apache.spark.sql.types.BooleanType; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.IntegerType; +import org.apache.spark.sql.types.MapType; import org.apache.spark.sql.types.StringType; -import org.apache.spark.sql.types.*; -import org.hl7.fhir.r4.model.*; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.types.TimestampType; +import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.MedicationRequest; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Questionnaire; +import org.hl7.fhir.r4.model.QuestionnaireResponse; import org.junit.Before; import org.junit.Test; import scala.collection.JavaConverters; @@ -382,4 +392,32 @@ public void testRestrictsOpenTypesCorrectly() { .filter(fn -> fn.startsWith("value")).collect(Collectors.toUnmodifiableSet()); assertEquals(Set.of("valueBoolean", "valueInteger", "valueCoding"), actualOpenTypeFieldNames); } + + @Test + public void testQuantity() { + final DataType quantityType = getField(observationSchema, true, "valueQuantity"); + + assertTrue(getField(quantityType, true, "value") instanceof DecimalType); + assertTrue(getField(quantityType, true, "value_scale") instanceof IntegerType); + assertTrue(getField(quantityType, true, "comparator") instanceof StringType); + assertTrue(getField(quantityType, true, "unit") instanceof StringType); + assertTrue(getField(quantityType, true, "system") instanceof StringType); + assertTrue(getField(quantityType, true, "code") instanceof StringType); + assertTrue(getField(quantityType, true, "value_canonicalized") instanceof DecimalType); + assertTrue(getField(quantityType, true, "code_canonicalized") instanceof StringType); + } + + @Test + public void testSimpleQuantity() { + final DataType quantityType = getField(medRequestSchema, true, "dispenseRequest", "quantity"); + + assertTrue(getField(quantityType, true, "value") instanceof DecimalType); + assertTrue(getField(quantityType, true, "value_scale") instanceof IntegerType); + assertTrue(getField(quantityType, true, "comparator") instanceof StringType); + assertTrue(getField(quantityType, true, "unit") instanceof StringType); + assertTrue(getField(quantityType, true, "system") instanceof StringType); + assertTrue(getField(quantityType, true, "code") instanceof StringType); + assertTrue(getField(quantityType, true, "value_canonicalized") instanceof DecimalType); + assertTrue(getField(quantityType, true, "code_canonicalized") instanceof StringType); + } } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java index 30802d415a..b755d857bd 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java @@ -162,6 +162,19 @@ public static Observation newObservation() { return observation; } + public static Observation newUcumObservation() { + final Observation observation = new Observation(); + observation.setId("blood-pressure-ucum"); + + final Quantity quantity = new Quantity(); + quantity.setValue(109); + quantity.setUnit("mmHg"); + quantity.setSystem("http://unitsofmeasure.org"); + quantity.setCode("mm[Hg]"); + + return observation; + } + /** * Returns a FHIR Patient for testing purposes. */ diff --git a/fhir-server/src/main/resources/tx/ucum-essence.xml b/encoders/src/test/resources/tx/ucum-essence.xml similarity index 100% rename from fhir-server/src/main/resources/tx/ucum-essence.xml rename to encoders/src/test/resources/tx/ucum-essence.xml diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 1f67a47533..40d5d9ae3d 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -6,8 +6,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -148,13 +148,6 @@ ${pathling.antlrVersion} - - - org.fhir - ucum - 1.0.3 - - com.amazonaws diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 407d8dd5dd..58d376f7b0 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -9,13 +9,13 @@ import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.struct; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.element.QuantityPath; -import au.csiro.pathling.terminology.ucum.Ucum; import java.math.BigDecimal; import java.util.Optional; import java.util.function.Function; diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java index 9414c2f059..1d39d03cc7 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java @@ -8,6 +8,7 @@ import static au.csiro.pathling.utilities.Preconditions.checkNotNull; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.literal.BooleanLiteralPath; @@ -30,7 +31,6 @@ import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.QuantityLiteralContext; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.StringLiteralContext; import au.csiro.pathling.fhirpath.parser.generated.FhirPathParser.TimeLiteralContext; -import au.csiro.pathling.terminology.ucum.Ucum; import java.text.ParseException; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java index f09318199d..1530c9cf9b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java @@ -7,6 +7,7 @@ package au.csiro.pathling.terminology.ucum; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhirpath.element.DecimalPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java deleted file mode 100644 index 63e0b37aef..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/Ucum.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.terminology.ucum; - -import java.io.InputStream; -import javax.annotation.Nonnull; -import org.fhir.ucum.UcumEssenceService; -import org.fhir.ucum.UcumException; -import org.fhir.ucum.UcumService; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -/** - * Makes UCUM services available to the rest of the application. - * - * @author John Grimes - */ -@Component -@Profile({"core", "fhir"}) -public class Ucum { - - public static final String SYSTEM_URI = "http://unitsofmeasure.org"; - - @Bean - @Nonnull - public static UcumService ucumEssenceService() throws UcumException { - final InputStream essenceStream = Ucum.class.getClassLoader() - .getResourceAsStream("tx/ucum-essence.xml"); - return new UcumEssenceService(essenceStream); - } - -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 20030a52ed..1203d1db9d 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -9,6 +9,7 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.async.SparkListener; import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; @@ -31,7 +32,6 @@ import au.csiro.pathling.terminology.CodingToLiteral; import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.terminology.ucum.ComparableQuantity; -import au.csiro.pathling.terminology.ucum.Ucum; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; From 3d55704e5a13c41e15ef2c86e026dbcfe3529446 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 10 May 2022 12:33:19 +1000 Subject: [PATCH 26/83] Incomplete implementation of schema building and serialization --- .../pathling/encoders/SchemaConverter.scala | 12 +++++++-- .../pathling/encoders/SerializerBuilder.scala | 26 +++++++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala index 2f261bc23f..ed6f6c90f6 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala @@ -14,12 +14,13 @@ package au.csiro.pathling.encoders import au.csiro.pathling.encoders.ExtensionSupport.{EXTENSIONS_FIELD_NAME, FID_FIELD_NAME} -import au.csiro.pathling.encoders.datatypes.DataTypeMappings +import au.csiro.pathling.encoders.datatypes.{DataTypeMappings, DecimalCustomCoder} import au.csiro.pathling.schema.SchemaVisitor import au.csiro.pathling.schema.SchemaVisitor.isCollection import ca.uhn.fhir.context._ import org.apache.spark.sql.types._ import org.hl7.fhir.instance.model.api.{IBase, IBaseResource} +import org.hl7.fhir.r4.model.{Quantity, SimpleQuantity} /** * The schema processor for converting FHIR schemas to SQL schemas. @@ -53,7 +54,14 @@ private[encoders] class SchemaConverterProcessor(override val fhirContext: FhirC } override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[StructField]): DataType = { - StructType(fields ++ createFidField() ++ createExtensionField(definition)) + val updatedFields = definition.getImplementingClass match { + case _: Class[Quantity] | _: Class[SimpleQuantity] => fields ++ Seq( + StructField("value_canonicalized", DecimalCustomCoder.decimalType), + StructField("code_canonicalized", DataTypes.StringType) + ) + case _ => fields + } + StructType(updatedFields ++ createFidField() ++ createExtensionField(definition)) } override def buildElement(elementName: String, elementValue: DataType, elementDefinition: BaseRuntimeElementDefinition[_]): StructField = { diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala index a69bc2e4a8..632eaea0be 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala @@ -15,17 +15,18 @@ package au.csiro.pathling.encoders import au.csiro.pathling.encoders.ExtensionSupport.{EXTENSIONS_FIELD_NAME, FID_FIELD_NAME} import au.csiro.pathling.encoders.SerializerBuilderProcessor.{dataTypeToUtf8Expr, getChildExpression, objectTypeFor} -import au.csiro.pathling.encoders.datatypes.DataTypeMappings +import au.csiro.pathling.encoders.datatypes.{DataTypeMappings, DecimalCustomCoder} +import au.csiro.pathling.encoders.terminology.ucum.Ucum import au.csiro.pathling.schema.SchemaVisitor.isCollection import au.csiro.pathling.schema._ import ca.uhn.fhir.context.BaseRuntimeElementDefinition.ChildTypeEnum import ca.uhn.fhir.context._ import org.apache.spark.sql.catalyst.expressions.objects.{ExternalMapToCatalyst, Invoke, MapObjects, StaticInvoke} import org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStruct, Expression, If, IsNull, Literal} -import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType, ObjectType} +import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.hl7.fhir.instance.model.api.{IBaseDatatype, IBaseHasExtensions, IBaseResource} -import org.hl7.fhir.r4.model.{Base, Extension} +import org.hl7.fhir.r4.model.{Base, Extension, Quantity, SimpleQuantity} import org.hl7.fhir.utilities.xhtml.XhtmlNode import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` @@ -104,13 +105,27 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[(String, Expression)]): Expression = { - val allFields = if (supportsExtensions) { + val allFields: Seq[(String, Expression)] = if (supportsExtensions) { fields ++ createExtensionsFields(definition) } else { fields } - val struct = CreateNamedStruct(allFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) + val updatedFields: Seq[(String, Expression)] = definition.getImplementingClass match { + case _: Class[Quantity] | _: Class[SimpleQuantity] => { + val value = Invoke(expression, "getValue", ObjectType(classOf[BigDecimal])) + val code = Invoke(expression, "getCode", ObjectType(classOf[String])) + val canonicalizedValue = StaticInvoke(classOf[Ucum], DecimalCustomCoder.decimalType, "getCanonicalValue", Seq(value, code)) + val canonicalizedCode = StaticInvoke(classOf[Ucum], DataTypes.StringType, "getCanonicalCode", Seq(value, code)) + allFields ++ Seq( + ("value_canonicalized", canonicalizedValue), + ("code_canonicalized", canonicalizedCode) + ) + } + case _ => allFields + } + + val struct = CreateNamedStruct(updatedFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) If(IsNull(expression), Literal.create(null, struct.dataType), struct) } @@ -263,6 +278,7 @@ private[encoders] object SerializerBuilderProcessor { case _ => childrenExts } } + flattenBase(composite).toMap } From 68dc126b5f540606309c1a1c71da014cd9aba979 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 10 May 2022 17:37:31 +1000 Subject: [PATCH 27/83] Added working implementation of Quantity serializer with canonical value of code. --- .../pathling/encoders/SerializerBuilder.scala | 47 ++++++++++++------- .../encoders/LightweightFhirEncodersTest.java | 9 ++-- .../au/csiro/pathling/encoders/TestData.java | 10 ++-- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala index 632eaea0be..1a85a97ad1 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStr import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.hl7.fhir.instance.model.api.{IBaseDatatype, IBaseHasExtensions, IBaseResource} -import org.hl7.fhir.r4.model.{Base, Extension, Quantity, SimpleQuantity} +import org.hl7.fhir.r4.model.{Base, Extension, Quantity} import org.hl7.fhir.utilities.xhtml.XhtmlNode import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` @@ -100,7 +100,33 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr } override def proceedCompositeChildren(value: CompositeCtx[Expression, (String, Expression)]): Seq[(String, Expression)] = { - dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition).getOrElse(super.proceedCompositeChildren(value)) + + value.compositeDefinition.getImplementingClass match { + case cls if classOf[Quantity].isAssignableFrom(cls) => { + val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) + val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) + // TODO: Manbe create specialized UCAM functions retuning spark.Decimal and UTF8String + // TODO: Or Maybe use the custom encoder here directly - but do we need to encode scale here as well? + // TODO: Also will this work for arrays ??? + // TODO: Also maybe start the fields with '_' so that they are removed from results (marking and synthetic fields) + // TODO: Maybe move to overrideCompositeExpression() providing it with a callback to generate default fields on request (a lazy call to super) + val canonicalizedValue = StaticInvoke(classOf[Decimal], + DecimalCustomCoder.decimalType, + "apply", + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) + val canonicalizedCode = + StaticInvoke( + classOf[UTF8String], + DataTypes.StringType, + "fromString", + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), "getCanonicalCode", Seq(valueExp, codeExp)) :: Nil) + super.proceedCompositeChildren(value) ++ Seq( + ("value_canonicalized", canonicalizedValue), + ("code_canonicalized", canonicalizedCode) + ) + } + case _ => dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition).getOrElse(super.proceedCompositeChildren(value)) + } } override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[(String, Expression)]): Expression = { @@ -110,22 +136,7 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr } else { fields } - - val updatedFields: Seq[(String, Expression)] = definition.getImplementingClass match { - case _: Class[Quantity] | _: Class[SimpleQuantity] => { - val value = Invoke(expression, "getValue", ObjectType(classOf[BigDecimal])) - val code = Invoke(expression, "getCode", ObjectType(classOf[String])) - val canonicalizedValue = StaticInvoke(classOf[Ucum], DecimalCustomCoder.decimalType, "getCanonicalValue", Seq(value, code)) - val canonicalizedCode = StaticInvoke(classOf[Ucum], DataTypes.StringType, "getCanonicalCode", Seq(value, code)) - allFields ++ Seq( - ("value_canonicalized", canonicalizedValue), - ("code_canonicalized", canonicalizedCode) - ) - } - case _ => allFields - } - - val struct = CreateNamedStruct(updatedFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) + val struct = CreateNamedStruct(allFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) If(IsNull(expression), Literal.create(null, struct.dataType), struct) } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index 9ee0db774e..bba701b5a2 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import java.math.BigDecimal; @@ -222,12 +223,12 @@ public void testQuantityCanonicalization() { final Row quantityRow = observationRow.getStruct(observationRow.fieldIndex("valueQuantity")); final BigDecimal canonicalizedValue = quantityRow.getDecimal( - quantityRow.fieldIndex("canonicalized_value")); + quantityRow.fieldIndex("value_canonicalized")); final String canonicalizedCode = quantityRow.getString( - quantityRow.fieldIndex("canonicalized_code")); + quantityRow.fieldIndex("code_canonicalized")); - assertEquals(new BigDecimal("1.0"), canonicalizedValue); - assertEquals("kg", canonicalizedCode); + assertEquals(new BigDecimal("76000.000000"), canonicalizedValue); + assertEquals("g", canonicalizedCode); } } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java index b755d857bd..905ccdf230 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java @@ -164,13 +164,15 @@ public static Observation newObservation() { public static Observation newUcumObservation() { final Observation observation = new Observation(); - observation.setId("blood-pressure-ucum"); + observation.setId("weight"); final Quantity quantity = new Quantity(); - quantity.setValue(109); - quantity.setUnit("mmHg"); + quantity.setValue(76); + quantity.setUnit("kg"); quantity.setSystem("http://unitsofmeasure.org"); - quantity.setCode("mm[Hg]"); + quantity.setCode("kg"); + + observation.setValue(quantity); return observation; } From 1d11552a9fff4cc292b01f0b6980573ac27960b7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Wed, 11 May 2022 10:10:32 +1000 Subject: [PATCH 28/83] Fix match condition within schema builder --- .../scala/au/csiro/pathling/encoders/SchemaConverter.scala | 4 ++-- .../au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java | 2 +- .../java/au/csiro/pathling/test/UnitTestDependencies.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala index ed6f6c90f6..e26b9fd168 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala @@ -20,7 +20,7 @@ import au.csiro.pathling.schema.SchemaVisitor.isCollection import ca.uhn.fhir.context._ import org.apache.spark.sql.types._ import org.hl7.fhir.instance.model.api.{IBase, IBaseResource} -import org.hl7.fhir.r4.model.{Quantity, SimpleQuantity} +import org.hl7.fhir.r4.model.Quantity /** * The schema processor for converting FHIR schemas to SQL schemas. @@ -55,7 +55,7 @@ private[encoders] class SchemaConverterProcessor(override val fhirContext: FhirC override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[StructField]): DataType = { val updatedFields = definition.getImplementingClass match { - case _: Class[Quantity] | _: Class[SimpleQuantity] => fields ++ Seq( + case cls if classOf[Quantity].isAssignableFrom(cls) => fields ++ Seq( StructField("value_canonicalized", DecimalCustomCoder.decimalType), StructField("code_canonicalized", DataTypes.StringType) ) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java index 1d39d03cc7..e46a3aa418 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/parser/LiteralTermVisitor.java @@ -162,7 +162,7 @@ public FhirPath visitQuantityLiteral(@Nullable final QuantityLiteralContext ctx) final String fhirPath = String.format("%s %s", number, ucumUnit.getText()); try { return QuantityLiteralPath.fromUcumString(fhirPath, resultContext, - Ucum.ucumEssenceService()); + Ucum.service()); } catch (final UcumException e) { throw new RuntimeException(e); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 1203d1db9d..0615334743 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -120,7 +120,7 @@ static TerminologyServiceFactory terminologyClientFactory() { @ConditionalOnMissingBean @Nonnull static UcumService ucumService() throws UcumException { - return Ucum.ucumEssenceService(); + return Ucum.service(); } } From b9e0a2d4d70817d82099f21a817ef12a1e1b15dc Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 09:59:48 +1000 Subject: [PATCH 29/83] Add schema test for Quantity array --- .../encoders/SchemaConverterTest.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java index 5126bbc543..3b5b471489 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java @@ -37,6 +37,7 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.types.TimestampType; import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.Device; import org.hl7.fhir.r4.model.MedicationRequest; import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Questionnaire; @@ -80,6 +81,8 @@ public class SchemaConverterTest { private StructType questionnaireSchema; private StructType questionnaireResponseSchema; + private StructType deviceSchema; + /** * Traverses a DataType recursively passing all encountered StructTypes to the provided consumer. @@ -163,6 +166,7 @@ public void setUp() { medRequestSchema = converter_L0.resourceSchema(MedicationRequest.class); questionnaireSchema = converter_L0.resourceSchema(Questionnaire.class); questionnaireResponseSchema = converter_L0.resourceSchema(QuestionnaireResponse.class); + deviceSchema = converter_L0.resourceSchema(Device.class); } @Test @@ -396,21 +400,22 @@ public void testRestrictsOpenTypesCorrectly() { @Test public void testQuantity() { final DataType quantityType = getField(observationSchema, true, "valueQuantity"); - - assertTrue(getField(quantityType, true, "value") instanceof DecimalType); - assertTrue(getField(quantityType, true, "value_scale") instanceof IntegerType); - assertTrue(getField(quantityType, true, "comparator") instanceof StringType); - assertTrue(getField(quantityType, true, "unit") instanceof StringType); - assertTrue(getField(quantityType, true, "system") instanceof StringType); - assertTrue(getField(quantityType, true, "code") instanceof StringType); - assertTrue(getField(quantityType, true, "value_canonicalized") instanceof DecimalType); - assertTrue(getField(quantityType, true, "code_canonicalized") instanceof StringType); + assertQuantityType(quantityType); } @Test public void testSimpleQuantity() { final DataType quantityType = getField(medRequestSchema, true, "dispenseRequest", "quantity"); + assertQuantityType(quantityType); + } + + @Test + public void testQuantityArray() { + final DataType quantityType = getField(deviceSchema, true, "property", "valueQuantity"); + assertQuantityType(quantityType); + } + private void assertQuantityType(final DataType quantityType) { assertTrue(getField(quantityType, true, "value") instanceof DecimalType); assertTrue(getField(quantityType, true, "value_scale") instanceof IntegerType); assertTrue(getField(quantityType, true, "comparator") instanceof StringType); From 69051a40e42d7a89c6fdc3abc0def36f885524ba Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 10:30:01 +1000 Subject: [PATCH 30/83] Add serialization test for Quantity array --- .../encoders/LightweightFhirEncodersTest.java | 42 +++++++++++++++---- .../au/csiro/pathling/encoders/TestData.java | 19 +++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index bba701b5a2..7a8a042efa 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -20,11 +20,11 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import au.csiro.pathling.encoders.terminology.ucum.Ucum; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import org.apache.spark.sql.Row; @@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.encoders.RowEncoder; import org.hl7.fhir.r4.model.BaseResource; import org.hl7.fhir.r4.model.Condition; +import org.hl7.fhir.r4.model.Device; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.MolecularSequence; import org.hl7.fhir.r4.model.MolecularSequence.MolecularSequenceQualityRocComponent; @@ -222,13 +223,40 @@ public void testQuantityCanonicalization() { final Row observationRow = rowEncoder.createDeserializer().apply(serializedRow); final Row quantityRow = observationRow.getStruct(observationRow.fieldIndex("valueQuantity")); - final BigDecimal canonicalizedValue = quantityRow.getDecimal( - quantityRow.fieldIndex("value_canonicalized")); - final String canonicalizedCode = quantityRow.getString( - quantityRow.fieldIndex("code_canonicalized")); + assertQuantity(quantityRow, "76000.000000", "g"); + } + + @Test + public void testQuantityArray() { + final ExpressionEncoder encoder = fhirEncoders.of(Device.class); + final Device device = TestData.newDevice(); + + final ExpressionEncoder resolvedEncoder = EncoderUtils.defaultResolveAndBind( + encoder); + final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(device); + + final ExpressionEncoder rowEncoder = EncoderUtils.defaultResolveAndBind( + RowEncoder.apply(encoder.schema())); + final Row deviceRow = rowEncoder.createDeserializer().apply(serializedRow); + + final List properties = deviceRow.getList(deviceRow.fieldIndex("property")); + final Row propertyRow = properties.get(0); + final List quantityArray = propertyRow.getList(propertyRow.fieldIndex("valueQuantity")); - assertEquals(new BigDecimal("76000.000000"), canonicalizedValue); - assertEquals("g", canonicalizedCode); + final Row quantity1 = quantityArray.get(0); + assertQuantity(quantity1, "0.001000", "m"); + + final Row quantity2 = quantityArray.get(1); + assertQuantity(quantity2, "0.002000", "m"); } + private void assertQuantity(final Row quantity2, final String value, final String unit) { + final BigDecimal canonicalizedValue2 = quantity2.getDecimal( + quantity2.fieldIndex("value_canonicalized")); + final String canonicalizedCode2 = quantity2.getString( + quantity2.fieldIndex("code_canonicalized")); + + assertEquals(new BigDecimal(value), canonicalizedValue2); + assertEquals(unit, canonicalizedCode2); + } } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java index 905ccdf230..2f03e45706 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/TestData.java @@ -27,6 +27,8 @@ import org.hl7.fhir.r4.model.Condition.ConditionStageComponent; import org.hl7.fhir.r4.model.DateTimeType; import org.hl7.fhir.r4.model.DecimalType; +import org.hl7.fhir.r4.model.Device; +import org.hl7.fhir.r4.model.Device.DevicePropertyComponent; import org.hl7.fhir.r4.model.Encounter; import org.hl7.fhir.r4.model.Extension; import org.hl7.fhir.r4.model.IdType; @@ -177,6 +179,23 @@ public static Observation newUcumObservation() { return observation; } + public static Device newDevice() { + final Device device = new Device(); + device.setId("some-device"); + + final DevicePropertyComponent property = new DevicePropertyComponent(); + property.setType( + new CodeableConcept(new Coding("urn:example:abc", "12345", "Some property"))); + final Quantity quantity1 = new Quantity(null, 1.0, "http://unitsofmeasure.org", "mm", "mm"); + final Quantity quantity2 = new Quantity(null, 2.0, "http://unitsofmeasure.org", "mm", "mm"); + final List quantities = List.of(quantity1, quantity2); + property.setValueQuantity(quantities); + final List properties = List.of(property); + device.setProperty(properties); + + return device; + } + /** * Returns a FHIR Patient for testing purposes. */ From 03d02646cc3742c2542c80d528c4f0e52b9d1940 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 10:32:53 +1000 Subject: [PATCH 31/83] Prefix canonicalized fields with underscore --- .../scala/au/csiro/pathling/encoders/SchemaConverter.scala | 4 ++-- .../scala/au/csiro/pathling/encoders/SerializerBuilder.scala | 5 ++--- .../csiro/pathling/encoders/LightweightFhirEncodersTest.java | 4 ++-- .../java/au/csiro/pathling/encoders/SchemaConverterTest.java | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala index e26b9fd168..12ea892afe 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala @@ -56,8 +56,8 @@ private[encoders] class SchemaConverterProcessor(override val fhirContext: FhirC override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[StructField]): DataType = { val updatedFields = definition.getImplementingClass match { case cls if classOf[Quantity].isAssignableFrom(cls) => fields ++ Seq( - StructField("value_canonicalized", DecimalCustomCoder.decimalType), - StructField("code_canonicalized", DataTypes.StringType) + StructField("_value_canonicalized", DecimalCustomCoder.decimalType), + StructField("_code_canonicalized", DataTypes.StringType) ) case _ => fields } diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala index 1a85a97ad1..4ed98c4dd2 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala @@ -107,7 +107,6 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) // TODO: Manbe create specialized UCAM functions retuning spark.Decimal and UTF8String // TODO: Or Maybe use the custom encoder here directly - but do we need to encode scale here as well? - // TODO: Also will this work for arrays ??? // TODO: Also maybe start the fields with '_' so that they are removed from results (marking and synthetic fields) // TODO: Maybe move to overrideCompositeExpression() providing it with a callback to generate default fields on request (a lazy call to super) val canonicalizedValue = StaticInvoke(classOf[Decimal], @@ -121,8 +120,8 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr "fromString", StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), "getCanonicalCode", Seq(valueExp, codeExp)) :: Nil) super.proceedCompositeChildren(value) ++ Seq( - ("value_canonicalized", canonicalizedValue), - ("code_canonicalized", canonicalizedCode) + ("_value_canonicalized", canonicalizedValue), + ("_code_canonicalized", canonicalizedCode) ) } case _ => dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition).getOrElse(super.proceedCompositeChildren(value)) diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index 7a8a042efa..f9f818baeb 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -252,9 +252,9 @@ public void testQuantityArray() { private void assertQuantity(final Row quantity2, final String value, final String unit) { final BigDecimal canonicalizedValue2 = quantity2.getDecimal( - quantity2.fieldIndex("value_canonicalized")); + quantity2.fieldIndex("_value_canonicalized")); final String canonicalizedCode2 = quantity2.getString( - quantity2.fieldIndex("code_canonicalized")); + quantity2.fieldIndex("_code_canonicalized")); assertEquals(new BigDecimal(value), canonicalizedValue2); assertEquals(unit, canonicalizedCode2); diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java index 3b5b471489..1db6ae4706 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java @@ -422,7 +422,7 @@ private void assertQuantityType(final DataType quantityType) { assertTrue(getField(quantityType, true, "unit") instanceof StringType); assertTrue(getField(quantityType, true, "system") instanceof StringType); assertTrue(getField(quantityType, true, "code") instanceof StringType); - assertTrue(getField(quantityType, true, "value_canonicalized") instanceof DecimalType); - assertTrue(getField(quantityType, true, "code_canonicalized") instanceof StringType); + assertTrue(getField(quantityType, true, "_value_canonicalized") instanceof DecimalType); + assertTrue(getField(quantityType, true, "_code_canonicalized") instanceof StringType); } } From c864b92231167ea7a1c99c3691b7ed2daf83e2cf Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 10:36:22 +1000 Subject: [PATCH 32/83] Clean up fixed TODOs --- .../au/csiro/pathling/encoders/SerializerBuilder.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala index 4ed98c4dd2..be5bfdc248 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala @@ -102,12 +102,10 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr override def proceedCompositeChildren(value: CompositeCtx[Expression, (String, Expression)]): Seq[(String, Expression)] = { value.compositeDefinition.getImplementingClass match { - case cls if classOf[Quantity].isAssignableFrom(cls) => { + case cls if classOf[Quantity].isAssignableFrom(cls) => val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) - // TODO: Manbe create specialized UCAM functions retuning spark.Decimal and UTF8String - // TODO: Or Maybe use the custom encoder here directly - but do we need to encode scale here as well? - // TODO: Also maybe start the fields with '_' so that they are removed from results (marking and synthetic fields) + // TODO: Maybe create specialized UCUM functions returning spark.Decimal and UTF8String // TODO: Maybe move to overrideCompositeExpression() providing it with a callback to generate default fields on request (a lazy call to super) val canonicalizedValue = StaticInvoke(classOf[Decimal], DecimalCustomCoder.decimalType, @@ -123,7 +121,6 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, overr ("_value_canonicalized", canonicalizedValue), ("_code_canonicalized", canonicalizedCode) ) - } case _ => dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition).getOrElse(super.proceedCompositeChildren(value)) } } From 00350ee4c59329efa3e17b3f2f19b2dde1b69163 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 11:01:23 +1000 Subject: [PATCH 33/83] Move UCUM essence into the main resources directory --- encoders/src/{test => main}/resources/tx/ucum-essence.xml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename encoders/src/{test => main}/resources/tx/ucum-essence.xml (100%) diff --git a/encoders/src/test/resources/tx/ucum-essence.xml b/encoders/src/main/resources/tx/ucum-essence.xml similarity index 100% rename from encoders/src/test/resources/tx/ucum-essence.xml rename to encoders/src/main/resources/tx/ucum-essence.xml From 38d4da289db580a886cad2dcd5f41b639b21651b Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 16:23:31 +1000 Subject: [PATCH 34/83] Use pre-baked canonicalized Quantities within FHIRPath functions --- .../encoders/terminology/ucum/Ucum.java | 14 +- .../fhirpath/element/QuantityPath.java | 48 +++---- .../fhirpath/encoding/QuantityEncoding.java | 11 +- .../fhirpath/literal/QuantityLiteralPath.java | 55 ++++++-- .../terminology/ucum/ComparableQuantity.java | 133 ------------------ .../pathling/test/UnitTestDependencies.java | 22 +-- .../pathling/test/helpers/SparkHelpers.java | 25 +++- 7 files changed, 115 insertions(+), 193 deletions(-) delete mode 100644 fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java index af8c5949d0..90750a963f 100644 --- a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java +++ b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java @@ -54,11 +54,19 @@ public static UcumService service() throws UcumException { public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, @Nonnull final String code) { try { - final Pair result = getCanonicalForm(value, code); + @Nullable final Pair result = getCanonicalForm(value, code); if (result == null) { return null; } - return new BigDecimal(result.getValue().asDecimal()); + @Nullable final Decimal decimalValue = result.getValue(); + if (decimalValue == null) { + return null; + } + @Nullable final String stringValue = decimalValue.asDecimal(); + if (stringValue == null) { + return null; + } + return new BigDecimal(stringValue); } catch (final UcumException e) { throw new RuntimeException(e); } @@ -68,7 +76,7 @@ public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, public static String getCanonicalCode(@Nonnull final BigDecimal value, @Nonnull final String code) { try { - final Pair result = getCanonicalForm(value, code); + @Nullable final Pair result = getCanonicalForm(value, code); if (result == null) { return null; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index addf6d5ad1..fb86c88711 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -6,7 +6,7 @@ package au.csiro.pathling.fhirpath.element; -import static org.apache.spark.sql.functions.callUDF; +import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.struct; import static org.apache.spark.sql.functions.when; @@ -15,9 +15,9 @@ import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; -import au.csiro.pathling.terminology.ucum.ComparableQuantity; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.Function; @@ -56,14 +56,16 @@ public Function getComparison(@Nonnull final ComparisonOpera public static Function buildComparison(@Nonnull final FhirPath source, @Nonnull final ComparisonOperation operation) { return target -> { - final Column comparableSource = callUDF(ComparableQuantity.FUNCTION_NAME, - source.getValueColumn()); - final Column comparableTarget = callUDF(ComparableQuantity.FUNCTION_NAME, - target.getValueColumn()); - final Column sourceCode = comparableSource.getField("code"); - final Column targetCode = comparableTarget.getField("code"); - final Column sourceValue = comparableSource.getField("value"); - final Column targetValue = comparableTarget.getField("value"); + final Column comparableSource = source.getValueColumn(); + final Column comparableTarget = target.getValueColumn(); + final Column sourceCode = comparableSource.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column targetCode = comparableTarget.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column sourceValue = comparableSource.getField( + QuantityEncoding.CANONICALIZED_VALUE_COLUMN); + final Column targetValue = comparableTarget.getField( + QuantityEncoding.CANONICALIZED_VALUE_COLUMN); final Column compareValues = operation.getSparkFunction().apply(sourceValue, targetValue); return when(sourceCode.equalTo(targetCode), compareValues).otherwise(null); }; @@ -77,19 +79,13 @@ public boolean isComparableTo(@Nonnull final Class type) { @Nonnull @Override public Column getNumericValueColumn() { - final Column comparable = buildComparableValueColumn(this); - return comparable.getField("value"); + return getValueColumn().getField(QuantityEncoding.CANONICALIZED_VALUE_COLUMN); } @Nonnull @Override public Column getNumericContextColumn() { - return buildComparableValueColumn(this); - } - - @Nonnull - public static Column buildComparableValueColumn(@Nonnull final Numeric source) { - return callUDF(ComparableQuantity.FUNCTION_NAME, source.getValueColumn()); + return getValueColumn(); } @Nonnull @@ -104,23 +100,27 @@ public static Function buildMathOperation(@Nonnull fina @Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset, @Nonnull final FHIRDefinedType fhirType) { return target -> { - final Column sourceQuantity = source.getValueColumn(); final Column sourceComparable = source.getNumericValueColumn(); final Column sourceContext = source.getNumericContextColumn(); + final Column targetContext = target.getNumericContextColumn(); final Column resultColumn = operation.getSparkFunction() .apply(sourceComparable, target.getNumericValueColumn()); + final Column sourceCanonicalizedCode = sourceContext.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column targetCanonicalizedCode = targetContext.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); final Column resultStruct = struct( sourceContext.getField("id").as("id"), resultColumn.as("value"), - sourceContext.getField("value_scale").as("value_scale"), + lit(null).as("value_scale"), sourceContext.getField("comparator").as("comparator"), - sourceContext.getField("unit").as("unit"), + sourceCanonicalizedCode.as("unit"), sourceContext.getField("system").as("system"), - sourceContext.getField("code").as("code"), + sourceCanonicalizedCode.as("code"), + resultColumn.as(QuantityEncoding.CANONICALIZED_VALUE_COLUMN), + sourceCanonicalizedCode.as(QuantityEncoding.CANONICALIZED_CODE_COLUMN), sourceContext.getField("_fid").as("_fid") ); - final Column sourceCanonicalizedCode = source.getNumericContextColumn().getField("code"); - final Column targetCanonicalizedCode = target.getNumericContextColumn().getField("code"); final Column resultQuantityColumn = when(sourceCanonicalizedCode.equalTo(targetCanonicalizedCode), resultStruct) .otherwise(null); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index bf4ee1e304..1510ca9688 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -24,6 +24,9 @@ public class QuantityEncoding { + public static final String CANONICALIZED_VALUE_COLUMN = "_value_canonicalized"; + public static final String CANONICALIZED_CODE_COLUMN = "_code_canonicalized"; + @Nullable public static Row encode(@Nullable final Quantity quantity) { if (quantity == null) { @@ -80,10 +83,16 @@ public static StructType dataType() { final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); final StructField system = new StructField("system", DataTypes.StringType, true, metadata); final StructField code = new StructField("code", DataTypes.StringType, true, metadata); + final StructField canonicalizedValue = new StructField(CANONICALIZED_VALUE_COLUMN, + DataTypes.createDecimalType( + DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); + final StructField canonicalizedCode = new StructField(CANONICALIZED_CODE_COLUMN, + DataTypes.StringType, true, metadata); final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, metadata); return new StructType( - new StructField[]{id, value, valueScale, comparator, unit, system, code, fid}); + new StructField[]{id, value, valueScale, comparator, unit, system, code, canonicalizedValue, + canonicalizedCode, fid}); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 58d376f7b0..e5afd5d607 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -16,7 +16,10 @@ import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.element.QuantityPath; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; +import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.regex.Matcher; @@ -45,6 +48,13 @@ public class QuantityLiteralPath extends LiteralPath implements Compar private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); + private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("second", "s") + .put("seconds", "s") + .put("millisecond", "ms") + .put("milliseconds", "ms") + .build(); + protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Quantity literalValue) { super(dataset, idColumn, literalValue); @@ -143,16 +153,40 @@ public String getExpression() { @Nonnull @Override public Column buildValueColumn() { - final Quantity value = getValue(); - final Optional comparator = Optional.ofNullable(value.getComparator()); + final Quantity quantity = getValue(); + final Optional comparator = Optional.ofNullable(quantity.getComparator()); + final BigDecimal value = quantity.getValue(); + + final BigDecimal canonicalizedValue; + final String canonicalizedCode; + if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { + // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. + canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); + canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); + } else if (quantity.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI) && + CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { + // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the + // UCUM library to canonicalize the value and code. + final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); + canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); + canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); + } else { + // If it is neither a UCUM Quantity nor a calendar duration, it will not have a canonicalized + // form available. + canonicalizedValue = null; + canonicalizedCode = null; + } + return struct( - lit(value.getId()).as("id"), - lit(value.getValue()).as("value"), - lit(value.getValue().scale()).as("value_scale"), + lit(quantity.getId()).as("id"), + lit(value).as("value"), + lit(value.scale()).as("value_scale"), lit(comparator.map(QuantityComparator::toCode).orElse(null)).as("comparator"), - lit(value.getUnit()).as("unit"), - lit(value.getSystem()).as("system"), - lit(value.getCode()).as("code"), + lit(quantity.getUnit()).as("unit"), + lit(quantity.getSystem()).as("system"), + lit(quantity.getCode()).as("code"), + lit(canonicalizedValue).as(QuantityEncoding.CANONICALIZED_VALUE_COLUMN), + lit(canonicalizedCode).as(QuantityEncoding.CANONICALIZED_CODE_COLUMN), lit(null).as("_fid")); } @@ -170,14 +204,13 @@ public boolean isComparableTo(@Nonnull final Class type) { @Nonnull @Override public Column getNumericValueColumn() { - final Column comparable = QuantityPath.buildComparableValueColumn(this); - return comparable.getField("value"); + return getValueColumn().getField(QuantityEncoding.CANONICALIZED_VALUE_COLUMN); } @Nonnull @Override public Column getNumericContextColumn() { - return QuantityPath.buildComparableValueColumn(this); + return getValueColumn(); } @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java deleted file mode 100644 index 1530c9cf9b..0000000000 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/ucum/ComparableQuantity.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research - * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source - * Software Licence Agreement. - */ - -package au.csiro.pathling.terminology.ucum; - -import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; -import au.csiro.pathling.encoders.terminology.ucum.Ucum; -import au.csiro.pathling.fhirpath.element.DecimalPath; -import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; -import au.csiro.pathling.sql.udf.SqlFunction1; -import com.google.common.collect.ImmutableMap; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.Map; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import lombok.extern.slf4j.Slf4j; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.types.DataType; -import org.fhir.ucum.Decimal; -import org.fhir.ucum.Pair; -import org.fhir.ucum.UcumException; -import org.fhir.ucum.UcumService; -import org.hl7.fhir.r4.model.Quantity; -import org.hl7.fhir.r4.model.Quantity.QuantityComparator; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -@Component -@Profile("core") -@Slf4j -public class ComparableQuantity implements SqlFunction1 { - - @Nonnull - private final UcumService ucumService; - - private static final long serialVersionUID = 2317610455653365964L; - private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() - .put("second", "s") - .put("seconds", "s") - .put("millisecond", "ms") - .put("milliseconds", "ms") - .build(); - - /** - * The name of this function when used within SQL. - */ - public static final String FUNCTION_NAME = "comparable_quantity"; - - public ComparableQuantity(@Nonnull final UcumService ucumService) { - this.ucumService = ucumService; - } - - @Override - public String getName() { - return FUNCTION_NAME; - } - - @Override - public DataType getReturnType() { - return QuantityEncoding.dataType(); - } - - @Nullable - @Override - public Row call(@Nullable final Row row) throws Exception { - if (row == null) { - return null; - } - - final Quantity input = QuantityEncoding.decode(row); - - // If system and code are not populated, the Quantity will not be comparable. - if (!input.hasValue() || input.getSystem() == null || input.getCode() == null) { - return null; - } - - // If the Quantity has a comparator, it will not be comparable. - if (input.getComparator() != null && input.getComparator() != QuantityComparator.NULL) { - return null; - } - - final String resolvedCode; - if (input.getSystem().equals(Ucum.SYSTEM_URI)) { - resolvedCode = input.getCode(); - } else if (input.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI) && - CALENDAR_DURATION_TO_UCUM.containsKey(input.getCode())) { - // If it is a comparable calendar duration, convert it to UCUM. - resolvedCode = CALENDAR_DURATION_TO_UCUM.get(input.getCode()); - } else { - // If the Quantity is not UCUM or a comparable calendar duration, it is not comparable. - return null; - } - - // Use the UCUM library to get the canonical form of the Quantity. - final int maxPrecision = DecimalPath.getDecimalType().precision(); - final Decimal value = new Decimal(input.getValue().toPlainString(), maxPrecision); - @Nullable final Pair canonical; - try { - canonical = ucumService.getCanonicalForm(new Pair(value, resolvedCode)); - } catch (final UcumException e) { - // If there is a problem parsing the UCUM expression, the Quantity is not comparable. - return null; - } - - // If the canonical form is not complete, we can't compare the quantities. - if (canonical == null || canonical.getCode() == null || canonical.getValue() == null) { - return null; - } - - BigDecimal canonicalizedValue = new BigDecimal(canonical.getValue().asDecimal()); - if (canonicalizedValue.precision() > DecimalCustomCoder.precision()) { - return null; - } else if (canonicalizedValue.scale() > DecimalCustomCoder.scale()) { - canonicalizedValue = canonicalizedValue.setScale(DecimalCustomCoder.scale(), - RoundingMode.HALF_UP); - } - - // Create a new Quantity object with the canonicalized result. - final Quantity result = new Quantity(); - result.setValue(canonicalizedValue); - result.setUnit(canonical.getCode()); - result.setSystem(Ucum.SYSTEM_URI); - result.setCode(canonical.getCode()); - - return QuantityEncoding.encode(result); - } - -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 0615334743..112ddc0af8 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -9,7 +9,6 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.async.SparkListener; import au.csiro.pathling.encoders.FhirEncoders; -import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; @@ -27,20 +26,16 @@ import au.csiro.pathling.sql.dates.time.TimeGreaterThanOrEqualToFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanOrEqualToFunction; -import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; -import au.csiro.pathling.terminology.CodingToLiteral; import au.csiro.pathling.terminology.TerminologyService; -import au.csiro.pathling.terminology.ucum.ComparableQuantity; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; +import java.util.Collections; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; import org.apache.spark.sql.SparkSession; -import org.fhir.ucum.UcumException; -import org.fhir.ucum.UcumService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Profile; @@ -59,10 +54,7 @@ class UnitTestDependencies { @Nonnull static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, - @Nonnull final Optional sparkListener, - @Nonnull final UcumService ucumService) { - final List sqlFunction1 = List.of(new CodingToLiteral(), - new ComparableQuantity(ucumService)); + @Nonnull final Optional sparkListener) { final List sqlFunction2 = List.of(new DateTimeAddDurationFunction(), new DateTimeSubtractDurationFunction(), new DateAddDurationFunction(), new DateSubtractDurationFunction(), @@ -71,7 +63,8 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), new TimeGreaterThanFunction(), new TimeGreaterThanOrEqualToFunction(), new TimeLessThanFunction(), new TimeLessThanOrEqualToFunction()); - return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2); + return Spark.build(configuration, environment, sparkListener, Collections.emptyList(), + sqlFunction2); } @Bean @@ -116,11 +109,4 @@ static TerminologyServiceFactory terminologyClientFactory() { return new TestTerminologyServiceFactory(); } - @Bean - @ConditionalOnMissingBean - @Nonnull - static UcumService ucumService() throws UcumException { - return Ucum.service(); - } - } diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 900aed8171..bd2e99df14 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -10,6 +10,7 @@ import static org.apache.spark.sql.functions.col; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import java.math.BigDecimal; import java.math.RoundingMode; @@ -116,10 +117,16 @@ public static StructType quantityStructType() { final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); final StructField system = new StructField("system", DataTypes.StringType, true, metadata); final StructField code = new StructField("code", DataTypes.StringType, true, metadata); + final StructField canonicalizedCode = new StructField("_code_canonicalized", + DataTypes.StringType, true, metadata); + final StructField canonicalizedValue = new StructField("_value_canonicalized", + DataTypes.createDecimalType( + DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, metadata); return new StructType( - new StructField[]{id, value, valueScale, comparator, unit, system, code, fid}); + new StructField[]{id, value, valueScale, comparator, unit, system, code, canonicalizedValue, + canonicalizedCode, fid}); } @Nonnull @@ -163,11 +170,23 @@ public static Row rowFromCodeableConcept(@Nonnull final CodeableConcept codeable @Nonnull public static Row rowFromQuantity(@Nonnull final Quantity quantity) { + final BigDecimal value = quantity.getValue(); + final String code = quantity.getCode(); + final BigDecimal canonicalizedValue; + final String canonicalizedCode; + if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { + canonicalizedValue = Ucum.getCanonicalValue(value, code); + canonicalizedCode = Ucum.getCanonicalCode(value, code); + } else { + canonicalizedValue = null; + canonicalizedCode = null; + } final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); return new GenericRowWithSchema( - new Object[]{quantity.getId(), quantity.getValue(), quantity.getValue().scale(), comparator, - quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */}, + new Object[]{quantity.getId(), quantity.getValue(), null /* scale */, comparator, + quantity.getUnit(), quantity.getSystem(), quantity.getCode(), canonicalizedValue, + canonicalizedCode, null /* _fid */}, quantityStructType()); } From 6e9f049e22ec1a688db59ae201cb0cd6dad9d308 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 16:38:29 +1000 Subject: [PATCH 35/83] Fix unit test dependencies --- .../pathling/test/UnitTestDependencies.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 112ddc0af8..87a125ad6b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -9,6 +9,7 @@ import au.csiro.pathling.Configuration; import au.csiro.pathling.async.SparkListener; import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; @@ -26,16 +27,19 @@ import au.csiro.pathling.sql.dates.time.TimeGreaterThanOrEqualToFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanFunction; import au.csiro.pathling.sql.dates.time.TimeLessThanOrEqualToFunction; +import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; +import au.csiro.pathling.terminology.CodingToLiteral; import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; -import java.util.Collections; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; import org.apache.spark.sql.SparkSession; +import org.fhir.ucum.UcumException; +import org.fhir.ucum.UcumService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Profile; @@ -55,6 +59,7 @@ class UnitTestDependencies { static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener) { + final List sqlFunction1 = List.of(new CodingToLiteral()); final List sqlFunction2 = List.of(new DateTimeAddDurationFunction(), new DateTimeSubtractDurationFunction(), new DateAddDurationFunction(), new DateSubtractDurationFunction(), @@ -63,8 +68,7 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), new TimeGreaterThanFunction(), new TimeGreaterThanOrEqualToFunction(), new TimeLessThanFunction(), new TimeLessThanOrEqualToFunction()); - return Spark.build(configuration, environment, sparkListener, Collections.emptyList(), - sqlFunction2); + return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2); } @Bean @@ -109,4 +113,11 @@ static TerminologyServiceFactory terminologyClientFactory() { return new TestTerminologyServiceFactory(); } + @Bean + @ConditionalOnMissingBean + @Nonnull + static UcumService ucumService() throws UcumException { + return Ucum.service(); + } + } From 1f23efb89009ee4cbd52c013607603304b5edf3c Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 16:49:52 +1000 Subject: [PATCH 36/83] Return null on UCUM exception, rather than throw These exceptions are thrown when the input is invalid. --- .../au/csiro/pathling/encoders/terminology/ucum/Ucum.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java index 90750a963f..18962f3b3b 100644 --- a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java +++ b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java @@ -68,7 +68,7 @@ public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, } return new BigDecimal(stringValue); } catch (final UcumException e) { - throw new RuntimeException(e); + return null; } } @@ -82,7 +82,7 @@ public static String getCanonicalCode(@Nonnull final BigDecimal value, } return result.getCode(); } catch (final UcumException e) { - throw new RuntimeException(e); + return null; } } From dbd8b61d004f22bbcba36ed5ffbbb1856618cef7 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 20 May 2022 17:39:58 +1000 Subject: [PATCH 37/83] Fix QuantityParserTest#lengthObservationSubtraction --- .../au/csiro/pathling/fhirpath/parser/QuantityParserTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java index 0bca573717..0230f3dd3a 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java @@ -50,9 +50,7 @@ void lengthObservationSubtraction() { assertThatResultOf("valueQuantity > (valueQuantity - 2 'g/dL')") .isElementPath(BooleanPath.class) .selectResult() - .saveAllRowsToCsv(spark, - "/Users/gri306/Code/pathling/fhir-server/src/test/resources/responses/ParserTest", - "lengthObservationSubtraction"); + .hasRows(spark, "responses/ParserTest/lengthObservationSubtraction.csv"); } } From c29710b254dd37112d84efabe5e90c79a87f2c8a Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 21 May 2022 18:22:41 +1000 Subject: [PATCH 38/83] Implement UntilFunction --- .../fhirpath/function/NamedFunction.java | 1 + .../fhirpath/function/UntilFunction.java | 62 +++++++++++++ .../java/au/csiro/pathling/spark/Spark.java | 7 +- .../sql/dates/TemporalDifferenceFunction.java | 90 +++++++++++++++++++ .../csiro/pathling/sql/udf/SqlFunction3.java | 13 +++ 5 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/NamedFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/NamedFunction.java index cb379a8cb1..d2b7a83ccb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/NamedFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/NamedFunction.java @@ -51,6 +51,7 @@ public interface NamedFunction { .put("allTrue", new BooleansTestFunction(ALL_TRUE)) .put("allFalse", new BooleansTestFunction(ALL_FALSE)) .put("extension", new ExtensionFunction()) + .put("until", new UntilFunction()) .build(); /** diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java new file mode 100644 index 0000000000..865fba0eee --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.function; + +import static au.csiro.pathling.QueryHelpers.join; +import static au.csiro.pathling.utilities.Preconditions.checkUserInput; +import static org.apache.spark.sql.functions.callUDF; + +import au.csiro.pathling.QueryHelpers.JoinType; +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.element.DatePath; +import au.csiro.pathling.fhirpath.element.DateTimePath; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.literal.DateLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; +import au.csiro.pathling.fhirpath.literal.StringLiteralPath; +import au.csiro.pathling.sql.dates.TemporalDifferenceFunction; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; + +public class UntilFunction implements NamedFunction { + + private static final String NAME = "until"; + + @Nonnull + @Override + public FhirPath invoke(@Nonnull final NamedFunctionInput input) { + checkUserInput(input.getArguments().size() == 2, + "until function must have two argument"); + final NonLiteralPath fromArgument = input.getInput(); + final FhirPath toArgument = input.getArguments().get(0); + final FhirPath calendarDurationArgument = input.getArguments().get(1); + + checkUserInput(fromArgument instanceof DateTimePath || fromArgument instanceof DatePath, + "until function must be invoked on a DateTime or Date"); + checkUserInput(toArgument instanceof DateTimePath || toArgument instanceof DateTimeLiteralPath + || toArgument instanceof DatePath || toArgument instanceof DateLiteralPath, + "until function must have a DateTime or Date as the first argument"); + checkUserInput(calendarDurationArgument instanceof StringLiteralPath, + "until function must have a String as the second argument"); + + final Dataset dataset = join(input.getContext(), fromArgument, toArgument, + JoinType.LEFT_OUTER); + final Column valueColumn = callUDF(TemporalDifferenceFunction.FUNCTION_NAME, + fromArgument.getValueColumn(), toArgument.getValueColumn(), + calendarDurationArgument.getValueColumn()); + final String expression = NamedFunction.expressionFromInput(input, NAME); + + return ElementPath.build(expression, dataset, fromArgument.getIdColumn(), + fromArgument.getEidColumn(), valueColumn, fromArgument.isSingular(), + fromArgument.getCurrentResource(), fromArgument.getThisColumn(), FHIRDefinedType.INTEGER); + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index eba2939125..6aab6df60c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -12,6 +12,7 @@ import au.csiro.pathling.sql.SqlStrategy; import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; +import au.csiro.pathling.sql.udf.SqlFunction3; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -53,7 +54,8 @@ public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, @Nonnull final List sqlFunction1, - @Nonnull final List sqlFunction2) { + @Nonnull final List sqlFunction2, + @Nonnull final List sqlFunction3) { log.debug("Creating Spark session"); resolveSparkConfiguration(environment); @@ -70,6 +72,9 @@ public static SparkSession build(@Nonnull final Configuration configuration, for (final SqlFunction2 function : sqlFunction2) { spark.udf().register(function.getName(), function, function.getReturnType()); } + for (final SqlFunction3 function : sqlFunction3) { + spark.udf().register(function.getName(), function, function.getReturnType()); + } // Configure AWS driver and credentials. configureAwsDriver(configuration, spark); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java new file mode 100644 index 0000000000..38c3054241 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java @@ -0,0 +1,90 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import au.csiro.pathling.errors.InvalidUserInputError; +import au.csiro.pathling.sql.udf.SqlFunction3; +import com.google.common.collect.ImmutableMap; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Component +@Profile("core") +public class TemporalDifferenceFunction implements SqlFunction3 { + + private static final long serialVersionUID = -7306741471632636471L; + public static final String FUNCTION_NAME = "date_diff"; + + static final Map CALENDAR_DURATION_TO_TEMPORAL = new ImmutableMap.Builder() + .put("year", ChronoUnit.YEARS) + .put("years", ChronoUnit.YEARS) + .put("month", ChronoUnit.MONTHS) + .put("months", ChronoUnit.MONTHS) + .put("day", ChronoUnit.DAYS) + .put("days", ChronoUnit.DAYS) + .put("hour", ChronoUnit.HOURS) + .put("hours", ChronoUnit.HOURS) + .put("minute", ChronoUnit.MINUTES) + .put("minutes", ChronoUnit.MINUTES) + .put("second", ChronoUnit.SECONDS) + .put("seconds", ChronoUnit.SECONDS) + .put("millisecond", ChronoUnit.MILLIS) + .put("milliseconds", ChronoUnit.MILLIS) + .build(); + + @Override + public String getName() { + return FUNCTION_NAME; + } + + @Override + public DataType getReturnType() { + return DataTypes.LongType; + } + + @Nullable + @Override + public Long call(@Nullable final String encodedFrom, @Nullable final String encodedTo, + @Nullable final String calendarDuration) throws Exception { + if (encodedFrom == null || encodedTo == null) { + return null; + } else if (calendarDuration == null) { + throw new InvalidUserInputError("Calendar duration must be provided"); + } + + final TemporalUnit temporalUnit = CALENDAR_DURATION_TO_TEMPORAL.get(calendarDuration); + + if (temporalUnit == null) { + throw new InvalidUserInputError("Invalid calendar duration: " + calendarDuration); + } + + final ZonedDateTime from = parse(encodedFrom); + final ZonedDateTime to = parse(encodedTo); + + return from.until(to, temporalUnit); + } + + private ZonedDateTime parse(final @Nonnull String encodedFrom) { + try { + return ZonedDateTime.parse(encodedFrom); + } catch (final DateTimeParseException e) { + return LocalDate.parse(encodedFrom).atStartOfDay(ZoneId.of("UTC")); + } + } + +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java new file mode 100644 index 0000000000..034429a9ce --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java @@ -0,0 +1,13 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.udf; + +import org.apache.spark.sql.api.java.UDF3; + +public interface SqlFunction3 extends SqlFunction, UDF3 { + +} From 543ec10e328e3d205468c1ab1ef6b91a5b7ee6ba Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 23 May 2022 14:00:16 +1000 Subject: [PATCH 39/83] Add parser test for UntilFunction --- .../pathling/fhirpath/parser/ParserTest.java | 54 +++-- .../pathling/test/UnitTestDependencies.java | 6 +- .../ParserTest/testUntilFunction.csv | 217 ++++++++++++++++++ 3 files changed, 253 insertions(+), 24 deletions(-) create mode 100644 fhir-server/src/test/resources/responses/ParserTest/testUntilFunction.csv diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java index 41e0f41e4b..d0212f1d03 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java @@ -48,6 +48,7 @@ import au.csiro.pathling.test.fixtures.RelationBuilder; import au.csiro.pathling.test.helpers.TerminologyHelpers; import java.util.Collections; +import javax.annotation.Nonnull; import org.apache.spark.sql.types.DataTypes; import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Enumerations.ResourceType; @@ -694,18 +695,7 @@ void testTraversalIntoMissingOpenType() { @Test void testReverseResolveFollowingMonomorphicResolve() { - final ResourcePath subjectResource = ResourcePath - .build(fhirContext, database, ResourceType.ENCOUNTER, ResourceType.ENCOUNTER.toCode(), - true); - - final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) - .terminologyClientFactory(terminologyServiceFactory) - .database(database) - .inputContext(subjectResource) - .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) - .build(); - parser = new Parser(parserContext); - + setSubjectResource(ResourceType.ENCOUNTER); assertThatResultOf( "serviceProvider.resolve().reverseResolve(Encounter.serviceProvider).id") .isElementPath(StringPath.class) @@ -715,17 +705,7 @@ void testReverseResolveFollowingMonomorphicResolve() { @Test void testReverseResolveFollowingPolymorphicResolve() { - final ResourcePath subjectResource = ResourcePath - .build(fhirContext, database, ResourceType.ENCOUNTER, ResourceType.ENCOUNTER.toCode(), - true); - - final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) - .terminologyClientFactory(terminologyServiceFactory) - .database(database) - .inputContext(subjectResource) - .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) - .build(); - parser = new Parser(parserContext); + setSubjectResource(ResourceType.ENCOUNTER); mockEmptyResource(database, spark, fhirEncoders, ResourceType.GROUP); @@ -746,4 +726,32 @@ void testReverseResolveFollowingReverseResolve() { .hasRows(spark, "responses/ParserTest/testReverseResolveFollowingReverseResolve.csv"); } + @Test + void testUntilFunction() { + setSubjectResource(ResourceType.ENCOUNTER); + mockEmptyResource(database, spark, fhirEncoders, ResourceType.GROUP); + assertThatResultOf( + "subject.resolve().ofType(Patient).birthDate.until(%resource.period.start, 'years')") + .isElementPath(IntegerPath.class) + .selectResult() + .saveAllRowsToCsv(spark, + "/Users/gri306/Code/pathling/fhir-server/src/test/resources/responses/ParserTest", + "testUntilFunction"); + } + + @SuppressWarnings("SameParameterValue") + private void setSubjectResource(@Nonnull final ResourceType resourceType) { + final ResourcePath subjectResource = ResourcePath + .build(fhirContext, database, resourceType, resourceType.toCode(), + true); + + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .terminologyClientFactory(terminologyServiceFactory) + .database(database) + .inputContext(subjectResource) + .groupingColumns(Collections.singletonList(subjectResource.getIdColumn())) + .build(); + parser = new Parser(parserContext); + } + } diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 87a125ad6b..790a095a68 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -13,6 +13,7 @@ import au.csiro.pathling.fhir.TerminologyClient; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.spark.Spark; +import au.csiro.pathling.sql.dates.TemporalDifferenceFunction; import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; import au.csiro.pathling.sql.dates.date.DateSubtractDurationFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; @@ -29,6 +30,7 @@ import au.csiro.pathling.sql.dates.time.TimeLessThanOrEqualToFunction; import au.csiro.pathling.sql.udf.SqlFunction1; import au.csiro.pathling.sql.udf.SqlFunction2; +import au.csiro.pathling.sql.udf.SqlFunction3; import au.csiro.pathling.terminology.CodingToLiteral; import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.test.stubs.TestTerminologyServiceFactory; @@ -68,7 +70,9 @@ static SparkSession sparkSession(@Nonnull final Configuration configuration, new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), new TimeGreaterThanFunction(), new TimeGreaterThanOrEqualToFunction(), new TimeLessThanFunction(), new TimeLessThanOrEqualToFunction()); - return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2); + final List sqlFunction3 = List.of(new TemporalDifferenceFunction()); + return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2, + sqlFunction3); } @Bean diff --git a/fhir-server/src/test/resources/responses/ParserTest/testUntilFunction.csv b/fhir-server/src/test/resources/responses/ParserTest/testUntilFunction.csv new file mode 100644 index 0000000000..43e711ecb7 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/testUntilFunction.csv @@ -0,0 +1,217 @@ +0134901a-4a69-4b39-92ea-d6f48ec83c6e,19 +01ac1476-0c9c-4cd7-82a6-4ff526018e9a,56 +0250a217-a663-403b-a708-5d14eadf0c40,42 +02b993c0-b358-481c-b0d0-0767387feb9f,0 +0364073f-91d8-47a1-b8b0-107c6318a691,58 +04945715-8ad5-4d8f-bfda-ae26662a3610,28 +04d88eb4-fb1b-4fa1-802a-5624f4c61b32,49 +07bb3bb3-09e6-4abf-85f4-8ad113e7afa5,46 +0af4a77e-892e-4b3f-9110-4c5224782250,51 +0c315d37-c758-4cf5-a7cb-cbf712fa7dfd,50 +102e16c5-4920-4dad-b142-0b280b2aacad,56 +11dafd5c-85e7-4275-a147-89a63641e35e,1 +12b212d8-bc6e-415a-93b0-594381726668,44 +12df1bed-5472-4c48-8e88-2cbb3e5eab33,50 +151e3048-66ad-4411-8753-677877e3bf0a,43 +16a3408d-c927-4d02-a1ae-cad32419caab,56 +16d280a2-c61f-487f-9034-22e884158969,15 +17bcf2ea-d921-4715-91c5-6b15226b33d3,17 +18ebf5ea-b0a2-4333-9e9c-40217de809ff,43 +19502b5a-2031-487d-9d9c-2001f959408b,51 +1cb88e7a-3db6-4a88-9b68-b0e1492114bc,56 +1db1c71b-9eeb-4a98-abc1-eb699b38510c,25 +1f3443ee-d15e-4894-b18e-185cfadfcdea,48 +1fd714a6-48b4-425a-99b3-ba5c1a995cce,58 +202131ae-78bb-4104-89b0-fb90645760f6,0 +20fbcb6e-d793-4b05-8e29-8ff82572b0db,59 +21995497-0eb8-4c0b-8c23-e6a829f3d596,50 +21c6e0fc-bf47-4f80-b1ff-85b940445bdf,49 +224e538d-3622-4f5e-b772-211ed358d8de,48 +23780032-f3bb-4b40-af2e-3fa6b1677376,45 +24c4dd5b-748b-4165-a0cc-2867b76c2dc2,13 +2514cdf0-1216-4c5e-bfa8-0c11c20c6b93,48 +274a2e49-9170-4945-bca2-ab6ef4bded75,26 +2830e99a-e6b0-411b-a051-7ea1e9f815d1,56 +29d4e919-2bd2-44e6-bb8a-380a780f60ff,54 +2aff9edd-def2-487a-b435-a162e11a303c,11 +2bd766e5-60e4-4469-bb97-54f0503a1eb0,43 +2bddd98c-86e3-4ae0-9dc6-87da16ab6267,54 +2cd8b9e4-2881-45a4-b1a6-c3a38e9d59d5,50 +2d2e07ec-e697-4384-a679-867e93740921,40 +2de1f829-c9d2-4f22-bf39-536dcb82fc3e,48 +2e8eb256-84c9-499a-8bf6-bb39504373e1,51 +2ec33cf5-d22d-414b-bf5d-e4b4e9997c0f,50 +2ee2ab26-09fb-4e12-bf21-9fed3b4c38de,51 +2fc75f11-2097-4e1e-8390-dbff3e844b7a,3 +2ffd6142-b75b-406f-bc06-cdb1c02eaefc,44 +308eba53-29fa-489a-97dc-741b077841a7,49 +30ae4a13-0cc6-4d12-a5dd-fa9288efcc09,37 +31d4c712-75b1-4ecb-9d1a-ccc4b8453826,12 +3397a796-894d-409c-97de-a9e6f3f88198,50 +359cb842-c25b-429f-ba9b-d52f171e5631,47 +36160225-76a2-4cf1-8bf8-5a3d3f2ec9d7,49 +374fd699-6b74-4500-9d60-2473c7e0364f,54 +395019b6-84e8-4919-8b8b-ac64e4a6eade,43 +39efc9b8-a75f-43a3-98a2-5bdc1c8207d2,11 +3ddf46fe-b5be-456d-810f-ea6a7402236d,54 +4148ac36-1dcb-4e96-89cd-355e7e8f919b,54 +41d26b1c-57b9-408c-b955-bf0ee1db4809,55 +4215b5d0-bdd9-4222-a04a-81bb637d60af,56 +42a9a333-d09d-4b9e-af96-1f02af26bac3,60 +42aa48c7-68cc-4bee-9730-cff29f0e36c5,43 +4506aa76-9d0d-40e4-85bc-5735f74522a0,53 +45b25ced-6eed-4fca-8ec1-b7d32ea13efe,50 +45ec3b9e-eb82-48b4-b4b6-bd305afdf8b3,12 +48cc2275-a478-4ade-b076-cf38e1d16017,58 +498f4b18-bd4c-470d-9cde-2fca70ec8964,3 +4a464bb5-9373-4f1b-9c03-053c64797114,2 +4adcaf72-5241-4877-b61c-f34060576c50,50 +4cc605d0-d0c6-4de3-849a-f5f92b35d2a0,20 +4cd95073-6db3-4eb2-ac4b-0aacb182b352,53 +4cfe72fe-21a0-4201-87b6-ef93695d2495,1 +4db144d3-a596-4718-a50a-8ac9175ad386,56 +4e6b69c7-61f9-4ff5-8c33-ad400edb9b06,43 +4f14ca7f-5332-4263-a7b2-f0a838380309,46 +4f342a71-dec3-403e-970b-e4c21b9eb98b,55 +4f3d1e93-a28f-446e-91af-2baf0168b82b,45 +50eac25a-3761-4de3-8a69-aae52e658300,60 +51241857-a7a4-4517-96e3-21f797581f89,50 +522e2abe-ad96-4d1a-b5a5-748faa997531,0 +54c750aa-570a-4e8e-9a02-418781923e39,3 +55f17f44-287f-41aa-a1bc-d1c0104af36e,48 +56999896-e36b-4e63-a6b6-dbb85dfe1936,59 +578cc35c-0982-4d72-a729-b304c026f075,48 +5abe533b-ae5f-47ad-8746-f60caf7483c0,40 +5cedf18a-fc44-4c39-a80f-2106a0d76934,53 +5e1fe0f2-4517-462b-8287-7824f9a60f4f,58 +5f64131b-d410-449a-9de6-35415919fec5,46 +62059efc-7607-41c9-a3ba-70948f6a8a1d,54 +634f0889-3a83-484a-bcc8-86f48e333c7b,49 +67e5cf95-236b-4f75-abc5-034ca2966448,22 +67fc12fc-bb9f-40f1-9051-127a788b3081,39 +683714ad-5194-49e7-9b8f-4e306cf68ae1,46 +6954c5c2-dd77-490a-9152-f1d78eff913d,52 +6a5d8a68-bbe0-4408-ada5-a843ee6bd9b0,10 +6af73cae-2e4e-485f-a74d-6f4307eb3af3,52 +6c84348c-5065-4e89-9412-bfda023683f2,42 +6d47a2fe-3645-4c5c-bebe-1b822eff197f,50 +6e187f47-91a1-4217-a185-31b6f01db225,57 +720f7ed6-7b2e-41dc-a0e0-724a3332aa24,50 +721e4027-a080-40d8-bc4a-43cd33477611,26 +72788e7f-1bf1-40b5-a1f1-27c669dc7278,46 +72c611bb-471e-40f2-a6a8-0ae7b7b6b63e,45 +749babeb-6883-4bd4-92a5-bec52769071c,44 +75a09d4f-eef2-4404-a386-dfcb408ec9ed,18 +777aa5f2-2a09-4aed-92ea-912694e28b48,53 +79be3b69-23d8-4408-8f01-68d993e63b6c,34 +7ac56d5a-32a7-4b40-a956-356e3006faed,17 +7e6fe8ca-a95e-4102-8b18-e6e858cd15a8,49 +7e7322a6-7912-4101-b3be-d134245a0626,55 +7ea43fc6-b1f7-46be-a308-5ecb275a1081,40 +7f410064-638a-4477-85c4-97fc2bb14a49,2 +8089a9ac-9e71-4487-a231-f720e4bc8d3e,50 +836b2e59-fb97-4014-ab0c-d5a5dc750969,0 +85481b3f-c75e-40cd-bacd-b0afb79e893c,50 +85c997bf-63d9-4ebb-8e0b-320de3dddc6c,48 +85e41a01-04e5-44d2-8b3a-4d3ad7b0f87b,42 +86415df5-7e47-4637-9e09-aaad9e7628d1,46 +8743e69b-17aa-44ff-b8fa-4f582665efc6,8 +87b04a97-1d33-4548-aec9-3d2856070702,56 +88b16960-bfff-41d0-81e4-9a4630834a00,2 +89f260bb-68b4-4dec-91f4-d52e673e0ff7,43 +8a1908f7-47cc-4f82-859c-781a193e9901,54 +8bc7af32-e5ad-4849-ba4d-b446db833ab4,44 +8be81657-8ba6-4ec5-8ff9-4809367096ea,54 +8dc2ce26-aec7-43c0-9771-350af4257ad8,50 +8e9e848c-0547-4f2c-8b5f-e33d79bbed67,3 +8f76d729-9f74-4cfd-be2a-8b033b568e18,18 +900ce9fe-02d3-476b-902a-9467767ecdcf,46 +922e2443-9cc5-421f-8310-9cd23f6e9f2d,48 +96b2282d-1384-4cfb-9958-f009fb501626,18 +97b42d67-8691-433d-8a31-dada076162ae,46 +984067fc-3dfc-47b7-8ee0-4d9b27d43860,44 +990c21ab-433b-4920-873e-f9dfc7103d9d,49 +99fc3558-79f3-4d14-9aa1-ff63f621ecd9,45 +9b2eb632-6f1a-4e97-912b-3c8378a1b11c,42 +9b4f52b0-51fa-4ea6-b02c-2ed8478066d3,51 +9bc516d4-7c82-4340-a90c-bb493f96fbe4,53 +9e0b8dc3-65c4-490d-97c0-d5e3db944de5,56 +9ed8703e-3b40-424c-9e98-b3b404051f99,52 +a1228f20-7c50-4d3a-b88c-2d277ca79d79,9 +a221198d-000e-497b-8b70-bb4d37f7bbe1,6 +a2c3ee1d-ec35-4b26-9bab-12de3f47d604,40 +a6790cbd-8349-432e-a976-5dfc07451203,44 +a6b084fb-ada7-480a-9adc-f180d4eb1e2a,50 +a6cb423a-7571-46df-83e2-ccc6820adb36,46 +a89c0c42-c19f-4d7f-a15e-d04e043c92f6,0 +acb2329d-b1c3-45c3-ad28-91a09aa521e1,40 +add6d754-a075-4693-a33a-c061c9a368ff,58 +ae7ddaef-4911-418c-8401-d978617e145b,23 +ae9e663e-8c15-43c9-8a88-52b61cbf07a9,56 +b00df815-7528-4967-bfe2-311130d91c21,26 +b07fb98d-2da4-423a-83b4-7a673de93096,48 +b0d337c5-3555-4817-ba59-4ceea5ad8a91,31 +b1ccd55a-6b88-4240-b20c-ca893f36e23b,42 +b1d4dc41-07ae-44db-ae17-1df86c5a62cf,56 +b5d06aa6-b5c6-4782-a0f8-dbf4121e1baa,52 +b6900c21-d310-4fb4-8b8f-176a09c91989,4 +b7772635-1801-48e4-a442-f4aaa6860544,41 +b870d2cb-ea6a-4b67-8de5-ce1224a18bf2,48 +bac374c0-50e8-4a5c-947b-82a8e9a747a9,50 +baf48725-f0f9-4357-a71d-b1104910deae,50 +bb1a7816-4798-4f58-9ce4-c5bd3af682a8,61 +bbd0f936-0cb9-4cc9-858c-2926b2b1c1eb,2 +bf895c48-1d52-4780-8e9a-532d69356d28,18 +bfc93cb7-e53a-49f7-b86e-81d6a13cfdea,46 +c06a9f9b-90fa-4bd6-9581-0c6950f03e4a,44 +c3ec1f2e-1411-41e4-8370-acd5a8f68cf4,54 +c57d51a7-45de-41b9-92fc-efd0634effaf,5 +c5c6eed5-aec6-4b8a-b689-adbb219c2267,4 +c892b1d7-7485-407d-8883-54fb7fecebc1,1 +c90aae7e-5704-4e13-8794-41ab377193bd,58 +ca3e0486-fd6b-4a13-a741-3a47760b412d,5 +ca7f269d-3794-4994-8c46-d8e9f2aa6378,56 +cc56af6d-25c6-480e-9767-da02a55551da,56 +cdb8493c-e3b0-447f-b16b-e58dd64660f3,55 +cf67a8d4-48e2-4dc9-a521-6aabcff0330b,56 +cfcbe34a-edc5-4442-bc05-5927fa00336b,55 +d05461d6-13bf-402c-890d-db6404933f7c,49 +d080c116-681a-494a-a928-45924109b49d,59 +d535b213-2abc-438f-aa6a-c06476d60b09,3 +d6988116-3c63-44f8-9f94-9e9d51cc5a37,52 +d79728e1-8834-4f4a-8edc-ce18aca18be6,60 +d7a48a26-7731-4046-bf22-78f0b8084eb9,43 +d90676d2-ce74-4867-a00f-6dbffa7c00be,56 +d9e7dd88-2d7c-4e48-8cf9-9e33f729b7c2,50 +db79f75d-af3c-4f82-b4e5-9b5d26598eef,42 +def5fd23-2b74-4c64-85e9-fac27e626bb7,18 +dfe38e2d-23d5-45c6-86bd-f83fe3b6a1d8,43 +e0dfbc08-45ad-4a81-b648-7e65c066f673,1 +e1040d58-53bc-4467-8101-ca53b772cede,45 +e1c2ad9f-6f61-4f16-805a-2f405b63659a,57 +e2387a81-5ff5-4daf-b2e8-881998d0d917,57 +e26ae29a-213a-4f79-98c2-cc81cf2f451d,1 +e363eb67-64c7-440f-93fd-5c8787c69a85,49 +e4358f28-62a8-4e06-b2bd-cb00d3310349,16 +e7f68d35-ae55-4fa4-b0be-0672a5a7186a,45 +e8c41168-a093-40eb-8baa-6802d24f554a,59 +e8eba267-fecb-477e-9625-771fbcfc3cba,56 +ede7745a-8f3e-4e20-9f16-33756be7ab6c,0 +ee24bf89-81a3-4259-a382-86917f3829d4,50 +f1293c5e-c997-4ba9-8bfd-438e7a840dc8,17 +f18b08bd-40da-43c1-87ec-4ba23542635f,44 +f19eefca-c1ad-4860-bdda-4674a631d464,28 +f28931e5-1f35-4f9f-a02e-78398735ea67,46 +f29330d0-5b32-4f75-b558-a1c7f05c0b4a,56 +f2c9ca81-8b1e-4873-bd8a-1a6e1411193c,30 +f3aa491a-4dbd-4436-b674-18b2177dcf18,54 +f3c2f842-6aa3-42c9-a0f3-895c40456868,0 +f3eb0c38-2571-44e7-be39-732eb52cf1df,1 +f6d3ff7b-f6e1-416a-bb05-73dfa8d129b5,47 +f77a8561-5adc-452a-ac9a-76273b6a6678,62 +f883f2ca-d523-4c9f-86b2-0add137901de,23 +fc4d37ec-22b7-4ea0-a34e-dcd4beb2001c,43 +fc771883-3bd6-46d9-9626-a130e1b902de,53 +fcab9efe-fb05-42d6-b366-ce6db1cc4093,14 +fcc916dd-5e4f-4bf7-9aef-c906a36d7301,50 +fce20464-ff98-4051-8714-6b9584e52740,7 From c5ad171b50482b7a2f9065de91919f64f8be4dcd Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 23 May 2022 22:03:31 +1000 Subject: [PATCH 40/83] Add UntilFunctionTest and TemporalDifferenceFunctionTest --- .../fhirpath/function/UntilFunction.java | 6 +- .../sql/dates/TemporalDifferenceFunction.java | 4 + .../fhirpath/function/UntilFunctionTest.java | 348 ++++++++++++++++++ .../dates/TemporalDifferenceFunctionTest.java | 43 +++ 4 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java index 865fba0eee..4ae551694c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java @@ -34,7 +34,7 @@ public class UntilFunction implements NamedFunction { @Override public FhirPath invoke(@Nonnull final NamedFunctionInput input) { checkUserInput(input.getArguments().size() == 2, - "until function must have two argument"); + "until function must have two arguments"); final NonLiteralPath fromArgument = input.getInput(); final FhirPath toArgument = input.getArguments().get(0); final FhirPath calendarDurationArgument = input.getArguments().get(1); @@ -46,6 +46,10 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { "until function must have a DateTime or Date as the first argument"); checkUserInput(calendarDurationArgument instanceof StringLiteralPath, "until function must have a String as the second argument"); + final String literalValue = ((StringLiteralPath) calendarDurationArgument).getValue() + .asStringValue(); + checkUserInput(TemporalDifferenceFunction.isValidCalendarDuration(literalValue), + "Invalid calendar duration: " + literalValue); final Dataset dataset = join(input.getContext(), fromArgument, toArgument, JoinType.LEFT_OUTER); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java index 38c3054241..a209533999 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java @@ -87,4 +87,8 @@ private ZonedDateTime parse(final @Nonnull String encodedFrom) { } } + public static boolean isValidCalendarDuration(final String literalValue) { + return CALENDAR_DURATION_TO_TEMPORAL.containsKey(literalValue); + } + } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java new file mode 100644 index 0000000000..f3822009c8 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java @@ -0,0 +1,348 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.function; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import au.csiro.pathling.errors.InvalidUserInputError; +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.NonLiteralPath; +import au.csiro.pathling.fhirpath.element.DatePath; +import au.csiro.pathling.fhirpath.element.DateTimePath; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.element.IntegerPath; +import au.csiro.pathling.fhirpath.element.ReferencePath; +import au.csiro.pathling.fhirpath.element.StringPath; +import au.csiro.pathling.fhirpath.literal.DateLiteralPath; +import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; +import au.csiro.pathling.fhirpath.literal.StringLiteralPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import ca.uhn.fhir.context.FhirContext; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import lombok.Value; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +/** + * @author John Grimes + */ +@SpringBootTest +@TestInstance(Lifecycle.PER_CLASS) +@Tag("UnitTest") +class UntilFunctionTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + static final String ID_ALIAS = "_abc123"; + static final String[] IDS = {"patient-1", "patient-2", "patient-3", "patient-4", "patient-5", + "patient-6"}; + + Dataset leftDataset() { + return new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-01T00:00:00Z") + .withRow("patient-2", "2020-01-01T00:00:00Z") + .withRow("patient-3", "2020-01-01") + .withRow("patient-4", null) + .withRow("patient-5", "2020-01-01T00:00:00Z") + .withRow("patient-6", null) + .build(); + } + + Dataset rightDataset() { + return new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2021-01-01T00:00:00Z") + .withRow("patient-2", "2021-01-01") + .withRow("patient-3", "2021-01-01T00:00:00Z") + .withRow("patient-4", "2021-01-01T00:00:00Z") + .withRow("patient-5", null) + .withRow("patient-6", null) + .build(); + } + + static final Object[] YEARS_RESULT = {1, 1, 1, null, null, null}; + static final Object[] MONTHS_RESULT = {12, 12, 12, null, null, null}; + static final Object[] DAYS_RESULT = {366, 366, 366, null, null, null}; + static final Object[] HOURS_RESULT = {8784, 8784, 8784, null, null, null}; + static final Object[] MINUTES_RESULT = {527040, 527040, 527040, null, null, null}; + static final Object[] SECONDS_RESULT = {31622400, 31622400, 31622400, null, null, null}; + private static final ImmutableMap CALENDAR_DURATION_TO_RESULT = new Builder() + .put("years", YEARS_RESULT) + .put("months", MONTHS_RESULT) + .put("days", DAYS_RESULT) + .put("hours", HOURS_RESULT) + .put("minutes", MINUTES_RESULT) + .put("seconds", SECONDS_RESULT) + .put("year", YEARS_RESULT) + .put("month", MONTHS_RESULT) + .put("day", DAYS_RESULT) + .put("hour", HOURS_RESULT) + .put("minute", MINUTES_RESULT) + .put("second", SECONDS_RESULT) + .build(); + + @Value + static class TestParameters { + + @Nonnull + String name; + + @Nonnull + NonLiteralPath input; + + @Nonnull + List arguments; + + @Nonnull + ParserContext context; + + @Nonnull + Dataset expectedResult; + + @Override + public String toString() { + return name; + } + + } + + @Nonnull + Stream parameters() { + final Collection parameters = new ArrayList<>(); + for (final String calendarDuration : CALENDAR_DURATION_TO_RESULT.keySet()) { + final ElementPath input = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATETIME) + .dataset(leftDataset()) + .idAndValueColumns() + .build(); + final ElementPath argument = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATETIME) + .dataset(rightDataset()) + .idAndValueColumns() + .build(); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(input.getIdColumn())) + .build(); + final Dataset expectedResult = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.IntegerType) + .withIdValueRows(Arrays.asList(IDS), id -> { + final int index = Integer.parseInt(id.split("-")[1]) - 1; + final Object[] results = CALENDAR_DURATION_TO_RESULT.get(calendarDuration); + assertNotNull(results); + return results[index]; + }).build(); + final List arguments = List.of(argument, + StringLiteralPath.fromString(calendarDuration, input)); + parameters.add( + new TestParameters(calendarDuration, input, arguments, context, expectedResult)); + } + return parameters.stream(); + } + + @ParameterizedTest + @MethodSource("parameters") + void test(@Nonnull final TestParameters parameters) { + final NamedFunctionInput input = new NamedFunctionInput(parameters.getContext(), + parameters.getInput(), parameters.getArguments()); + final FhirPath result = NamedFunction.getInstance("until").invoke(input); + assertThat(result) + .isElementPath(IntegerPath.class) + .selectResult() + .hasRows(parameters.getExpectedResult()); + } + + @Test + void milliseconds() { + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-01") + .build(); + final Dataset rightDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-02") + .build(); + final ElementPath input = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + final ElementPath argument = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(rightDataset) + .idAndValueColumns() + .build(); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(input.getIdColumn())) + .build(); + final Dataset expectedResult = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.IntegerType) + .withRow("patient-1", 86400000) + .build(); + final List arguments1 = List.of(argument, + StringLiteralPath.fromString("millisecond", input)); + final List arguments2 = List.of(argument, + StringLiteralPath.fromString("millisecond", input)); + for (final List arguments : List.of(arguments1, arguments2)) { + final NamedFunctionInput functionInput = new NamedFunctionInput(context, input, arguments); + final FhirPath result = NamedFunction.getInstance("until").invoke(functionInput); + assertThat(result) + .isElementPath(IntegerPath.class) + .selectResult() + .hasRows(expectedResult); + } + } + + @Test + void dateLiteralArgument() throws ParseException { + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-01") + .build(); + final ElementPath input = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + final DateLiteralPath argument = DateLiteralPath.fromString("2020-01-02", input); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(input.getIdColumn())) + .build(); + final Dataset expectedResult = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.IntegerType) + .withRow("patient-1", 86400000) + .build(); + final List arguments = List.of(argument, + StringLiteralPath.fromString("millisecond", input)); + final NamedFunctionInput functionInput = new NamedFunctionInput(context, input, arguments); + final FhirPath result = NamedFunction.getInstance("until").invoke(functionInput); + assertThat(result) + .isElementPath(IntegerPath.class) + .selectResult() + .hasRows(expectedResult); + } + + @Test + void dateTimeLiteralArgument() throws ParseException { + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-01") + .build(); + final ElementPath input = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + final DateTimeLiteralPath argument = DateTimeLiteralPath.fromString("2020-01-02T00:00:00Z", + input); + final ParserContext context = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(input.getIdColumn())) + .build(); + final Dataset expectedResult = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.IntegerType) + .withRow("patient-1", 86400000) + .build(); + final List arguments = List.of(argument, + StringLiteralPath.fromString("millisecond", input)); + final NamedFunctionInput functionInput = new NamedFunctionInput(context, input, arguments); + final FhirPath result = NamedFunction.getInstance("until").invoke(functionInput); + assertThat(result) + .isElementPath(IntegerPath.class) + .selectResult() + .hasRows(expectedResult); + } + + @Test + void invalidCalendarDuration() { + final Dataset leftDataset = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withColumn(DataTypes.StringType) + .withRow("patient-1", "2020-01-01") + .build(); + final ElementPath input = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.DATE) + .dataset(leftDataset) + .idAndValueColumns() + .build(); + final NamedFunctionInput functionInput = new NamedFunctionInput(mock(ParserContext.class), + input, + List.of(mock(DateTimePath.class), StringLiteralPath.fromString("nanosecond", input))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(functionInput)); + assertEquals("Invalid calendar duration: nanosecond", error.getMessage()); + } + + @Test + void wrongNumberOfArguments() { + final NamedFunctionInput input = new NamedFunctionInput(mock(ParserContext.class), + mock(DateTimePath.class), List.of(mock(DateTimePath.class))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(input)); + assertEquals("until function must have two arguments", error.getMessage()); + } + + @Test + void wrongInputType() { + final NamedFunctionInput input = new NamedFunctionInput(mock(ParserContext.class), + mock(StringPath.class), List.of(mock(DateTimePath.class), mock(StringLiteralPath.class))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(input)); + assertEquals("until function must be invoked on a DateTime or Date", error.getMessage()); + } + + @Test + void wrongArgumentType() { + final NamedFunctionInput input = new NamedFunctionInput(mock(ParserContext.class), + mock(DatePath.class), List.of(mock(ReferencePath.class), mock(StringLiteralPath.class))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(input)); + assertEquals("until function must have a DateTime or Date as the first argument", + error.getMessage()); + } + +} \ No newline at end of file diff --git a/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java b/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java new file mode 100644 index 0000000000..1968db3ef0 --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.sql.dates; + +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import au.csiro.pathling.errors.InvalidUserInputError; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("UnitTest") +class TemporalDifferenceFunctionTest { + + TemporalDifferenceFunction function; + + @BeforeEach + void setUp() { + function = new TemporalDifferenceFunction(); + } + + @Test + void nullCalendarDuration() { + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> function.call("2020-01-01", "2020-01-01", null)); + assertEquals("Calendar duration must be provided", + error.getMessage()); + } + + @Test + void invalidCalendarDuration() { + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> function.call("2020-01-01", "2020-01-01", "femtoseconds")); + assertEquals("Invalid calendar duration: femtoseconds", + error.getMessage()); + } + +} \ No newline at end of file From 501b443a9430e3e657cf0dfe342c93d3c795cbcc Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 24 May 2022 09:20:47 +1000 Subject: [PATCH 41/83] Add missing assertion within ParserTest#testUntilFunction --- .../java/au/csiro/pathling/fhirpath/parser/ParserTest.java | 4 +--- .../java/au/csiro/pathling/test/assertions/DatasetAssert.java | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java index d0212f1d03..ed039d8f40 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java @@ -734,9 +734,7 @@ void testUntilFunction() { "subject.resolve().ofType(Patient).birthDate.until(%resource.period.start, 'years')") .isElementPath(IntegerPath.class) .selectResult() - .saveAllRowsToCsv(spark, - "/Users/gri306/Code/pathling/fhir-server/src/test/resources/responses/ParserTest", - "testUntilFunction"); + .hasRows(spark, "responses/ParserTest/testUntilFunction.csv"); } @SuppressWarnings("SameParameterValue") diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/assertions/DatasetAssert.java b/fhir-server/src/test/java/au/csiro/pathling/test/assertions/DatasetAssert.java index 9bd6d2d98d..5e560a4d4e 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/assertions/DatasetAssert.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/assertions/DatasetAssert.java @@ -172,7 +172,8 @@ public DatasetAssert saveAllRowsToCsv(@Nonnull final SparkSession spark, log.error("Problem cleaning up", e); } - return this; + throw new AssertionError( + "Rows saved to CSV, check that the file is correct and replace this line with an assertion"); } private static class DeleteDirectoryVisitor extends SimplePathVisitor { From 932b46b11828930ea7b46965a6847beedc704e11 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 24 May 2022 19:28:35 +1000 Subject: [PATCH 42/83] Reformat POMs --- encoders/pom.xml | 4 ++-- fhir-server/pom.xml | 8 +++----- lib/import/pom.xml | 2 +- lib/js/pom.xml | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/encoders/pom.xml b/encoders/pom.xml index 4e1db646a5..0d52309bda 100644 --- a/encoders/pom.xml +++ b/encoders/pom.xml @@ -13,8 +13,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 40d5d9ae3d..68c376668a 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -6,8 +6,8 @@ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -335,9 +335,7 @@ ${project.version} - - -Xmx2g -XX:MaxMetaspaceSize=400m -XX:ReservedCodeCacheSize=240m -Xss1m -Duser.timezone=UTC - + -Xmx2g -XX:MaxMetaspaceSize=400m -XX:ReservedCodeCacheSize=240m -Xss1m -Duser.timezone=UTC Copyright © 2018-2022, Commonwealth Scientific and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source Software Licence Agreement. diff --git a/lib/import/pom.xml b/lib/import/pom.xml index de6dbac5e8..61e1bf8824 100644 --- a/lib/import/pom.xml +++ b/lib/import/pom.xml @@ -78,7 +78,7 @@ - + npmPublish diff --git a/lib/js/pom.xml b/lib/js/pom.xml index e7d1696950..2de3c9a3c0 100644 --- a/lib/js/pom.xml +++ b/lib/js/pom.xml @@ -59,7 +59,7 @@ - + npmPublish From 720d862e0c00d6290d8e9c346af70c85370fd03e Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Jul 2022 09:56:34 +1000 Subject: [PATCH 43/83] Switch Ucum-java to snapshot build, and fix referential problems --- encoders/pom.xml | 6 ++-- .../pathling/encoders/FhirEncodersTest.java | 2 +- fhir-server/pom.xml | 2 +- .../au/csiro/pathling/fhirpath/Numeric.java | 1 + .../fhirpath/element/DecimalPath.java | 9 +++-- .../java/au/csiro/pathling/spark/Spark.java | 3 ++ .../fhirpath/function/UntilFunctionTest.java | 4 +-- .../csiro/pathling/sql/SqlOperationsTest.java | 2 +- .../dates/TemporalDifferenceFunctionTest.java | 4 +-- lib/import/pom.xml | 2 +- lib/js/pom.xml | 2 +- lib/python/pom.xml | 2 +- library-api/pom.xml | 2 +- .../pathling/library/PathlingContext.java | 4 +-- pom.xml | 2 +- site/pom.xml | 2 +- terminology/pom.xml | 2 +- .../au/csiro/pathling/sql/SqlStrategy.scala | 33 ++++++++++++++++++- utilities/pom.xml | 2 +- 19 files changed, 60 insertions(+), 26 deletions(-) diff --git a/encoders/pom.xml b/encoders/pom.xml index df8eaf3337..82e097479c 100644 --- a/encoders/pom.xml +++ b/encoders/pom.xml @@ -20,7 +20,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT encoders jar @@ -98,9 +98,9 @@ - org.fhir + au.csiro.pathling ucum - 1.0.3 + 1.0.4-SNAPSHOT diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java index 91bd370333..0813c4a736 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/FhirEncodersTest.java @@ -543,6 +543,6 @@ public void testQuantityComparator() { final String queriedComparator = observationsDataset.select("valueQuantity.comparator").head() .getString(0); - Assert.assertEquals(originalComparator.toCode(), queriedComparator); + assertEquals(originalComparator.toCode(), queriedComparator); } } diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index eeb1257958..b095586d3e 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT fhir-server jar diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java index 5830709ce3..842b9ee86c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Numeric.java @@ -71,6 +71,7 @@ Function getMathOperation(@Nonnull MathOperation operat * Note that there can be multiple valid FHIR types for a given FHIRPath type, e.g. {@code uri} * and {@code code} both map to the {@code String} FHIRPath type. * + * @return the FHIR data type of the expression * @see Using FHIR types in expressions */ @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java index 06c397b149..2a334a8275 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java @@ -125,11 +125,10 @@ public Column getNumericContextColumn() { /** * Builds a math operation result for a Decimal-like path. * - * @param source The left operand for the operation - * @param operation The type of {@link au.csiro.pathling.fhirpath.Numeric.MathOperation} - * @param expression The FHIRPath expression to use in the result - * @param dataset The {@link Dataset} to use in the result - * @param fhirType The {@link FHIRDefinedType} to use in the result + * @param source the left operand for the operation + * @param operation the type of {@link au.csiro.pathling.fhirpath.Numeric.MathOperation} + * @param expression the FHIRPath expression to use in the result + * @param dataset the {@link Dataset} to use in the result * @return A {@link Function} that takes a {@link Numeric} as a parameter, and returns a * {@link NonLiteralPath} */ diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index 6aab6df60c..eff616bf89 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -45,6 +45,9 @@ public class Spark { * creation * @param environment Spring {@link Environment} from which to harvest Spark configuration * @param sparkListener a {@link SparkListener} that is used to monitor progress of jobs + * @param sqlFunction1 a list of {@link SqlFunction1} that should be registered + * @param sqlFunction2 a list of {@link SqlFunction2} that should be registered + * @param sqlFunction3 a list of {@link SqlFunction3} that should be registered * @return A shiny new {@link SparkSession} */ @Bean(destroyMethod = "stop") diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java index f3822009c8..2b0dc2d77b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java @@ -7,9 +7,9 @@ package au.csiro.pathling.fhirpath.function; import static au.csiro.pathling.test.assertions.Assertions.assertThat; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import au.csiro.pathling.errors.InvalidUserInputError; @@ -345,4 +345,4 @@ void wrongArgumentType() { error.getMessage()); } -} \ No newline at end of file +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java b/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java index d75172a240..2edbc39b07 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/sql/SqlOperationsTest.java @@ -81,7 +81,7 @@ void testMapWithPartitionPreview() { .withRow("patient-4", null, true) .build().repartition(1); - final Dataset resultDataset = SqlOperations.mapWithPartitionPreview(dataset, + final Dataset resultDataset = SqlExtensions.mapWithPartitionPreview(dataset, dataset.col("gender"), SqlOperationsTest::stringDecoder, new TestMapperWithPreview(), diff --git a/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java b/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java index 1968db3ef0..d09f416ae4 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunctionTest.java @@ -6,8 +6,8 @@ package au.csiro.pathling.sql.dates; -import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import au.csiro.pathling.errors.InvalidUserInputError; import org.junit.jupiter.api.BeforeEach; @@ -40,4 +40,4 @@ void invalidCalendarDuration() { error.getMessage()); } -} \ No newline at end of file +} diff --git a/lib/import/pom.xml b/lib/import/pom.xml index 7356bd41cc..ee4993d1d8 100644 --- a/lib/import/pom.xml +++ b/lib/import/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT ../../pom.xml import diff --git a/lib/js/pom.xml b/lib/js/pom.xml index 94d8bc1d81..39e739b66d 100644 --- a/lib/js/pom.xml +++ b/lib/js/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT ../../pom.xml js diff --git a/lib/python/pom.xml b/lib/python/pom.xml index 6c1fceec7e..2b9f8a5d40 100644 --- a/lib/python/pom.xml +++ b/lib/python/pom.xml @@ -7,7 +7,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT ../../pom.xml python diff --git a/library-api/pom.xml b/library-api/pom.xml index 926d2132ba..10b1a3529b 100644 --- a/library-api/pom.xml +++ b/library-api/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.3.1 + 5.4.0-SNAPSHOT library-api jar diff --git a/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java b/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java index 8e710b6d96..792cee8dc9 100644 --- a/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java +++ b/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java @@ -24,7 +24,7 @@ import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.fhir.DefaultTerminologyServiceFactory; import au.csiro.pathling.fhir.TerminologyServiceFactory; -import au.csiro.pathling.sql.PathlingStrategy; +import au.csiro.pathling.sql.SqlStrategy; import au.csiro.pathling.support.FhirConversionSupport; import au.csiro.pathling.terminology.TerminologyFunctions; import ca.uhn.fhir.context.FhirContext; @@ -73,7 +73,7 @@ private PathlingContext(@Nonnull final SparkSession spark, this.fhirVersion = fhirEncoders.getFhirVersion(); this.fhirEncoders = fhirEncoders; this.terminologyServiceFactory = terminologyServiceFactory; - PathlingStrategy.setup(spark); + SqlStrategy.setup(spark); } /** diff --git a/pom.xml b/pom.xml index 19a6a171e9..761ab07a28 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT pom Pathling diff --git a/site/pom.xml b/site/pom.xml index ffbefb6696..6215c6dda7 100644 --- a/site/pom.xml +++ b/site/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.3.1 + 5.4.0-SNAPSHOT ../pom.xml site diff --git a/terminology/pom.xml b/terminology/pom.xml index 014edfffe2..526e24cd82 100644 --- a/terminology/pom.xml +++ b/terminology/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.3.1 + 5.4.0-SNAPSHOT terminology jar diff --git a/terminology/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala b/terminology/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala index 3d8bbfae75..078ec5aa34 100644 --- a/terminology/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala +++ b/terminology/src/main/scala/au/csiro/pathling/sql/SqlStrategy.scala @@ -1,3 +1,34 @@ -package au.csiro.pathling.sql object SqlStrategy { +package au.csiro.pathling.sql +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.{SparkSession, Strategy} + +/** + * Custom spark SQL strategy with additional rules for custom Pathling operations. + */ +object SqlStrategy extends Strategy { + override def apply(plan: LogicalPlan): Seq[SparkPlan] = { + plan match { + + case MapWithPartitionPreview(serializer, decoder, deserializer, preview, mapper, child) => + MapWithPartitionPreviewExec(deserializer, decoder, serializer.value, preview, mapper, + planLater(child)) :: Nil + + case _ => Nil + } + } + + + /** + * Injects SqlStrategy into a given Spark session. + * + * @param session Spark session to add SqlStrategy to + */ + def setup(session: SparkSession): Unit = { + if (!session.experimental.extraStrategies.contains(SqlStrategy)) { + session.experimental.extraStrategies = Seq(SqlStrategy) ++ session.experimental + .extraStrategies + } + } } diff --git a/utilities/pom.xml b/utilities/pom.xml index 6c6ac78992..470028c57f 100644 --- a/utilities/pom.xml +++ b/utilities/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.3.1 + 5.4.0-SNAPSHOT utilities jar From 2e41aaeb335d40e265c2cb478e684b6205452709 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Jul 2022 10:00:40 +1000 Subject: [PATCH 44/83] Fix problem with Javadoc on PathlingContext.create --- .../src/main/java/au/csiro/pathling/library/PathlingContext.java | 1 + 1 file changed, 1 insertion(+) diff --git a/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java b/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java index 792cee8dc9..65d0a8959a 100644 --- a/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java +++ b/library-api/src/main/java/au/csiro/pathling/library/PathlingContext.java @@ -101,6 +101,7 @@ public static PathlingContext create(@Nonnull final SparkSession spark, * that all direct or indirect fields of type T in element of type T should be skipped. * @param enableExtensions switches on/off the support for FHIR extensions. * @param enabledOpenTypes list of types that are encoded within open types, such as extensions. + * @param terminologyServerUrl the URL of the terminology server to use * @return then new instance of PathlingContext. */ @Nonnull From 3e13c9d1714399833c5f9f07941d9bd059c1e909 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Jul 2022 10:16:51 +1000 Subject: [PATCH 45/83] Add snapshot repository to POM --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index 761ab07a28..4c95374985 100644 --- a/pom.xml +++ b/pom.xml @@ -797,6 +797,16 @@ + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots/ + + true + + + + ossrh From 6cd6def8045012ac3a2af4566bf013917a808782 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 11 Jul 2022 10:27:48 +1000 Subject: [PATCH 46/83] Fix trailing whitespace in DecimalCustomCoder --- .../csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala index ea60d2192f..d0e748dc27 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/datatypes/DecimalCustomCoder.scala @@ -115,7 +115,7 @@ object DecimalCustomCoder { * For location coordinates 6 decimal digits allow for location precision of 10cm, * so should be sufficient for any medical purpose. * - * So the final type is DECIMAL(32,6) which allows both for 6 decimal places and 26 digits + * So the final type is DECIMAL(32,6) which allows both for 6 decimal places and 26 digits * (regardless if there any decimal digits or not) */ From 88ed4d640375b5bb789909c2d5ecca841697dfd3 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Thu, 1 Sep 2022 11:35:44 +0200 Subject: [PATCH 47/83] Fixing the timezone for unit and integration tests to UTC. --- fhir-server/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index b095586d3e..d3817c07f2 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -314,6 +314,7 @@ true file://${project.basedir}/src/test/resources/test-data parquet + UTC @@ -331,6 +332,7 @@ file://${project.basedir}/src/test/resources/test-data parquet + UTC From 066ea8da04206f2ad23e54468d931cdba2b3d127 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 5 Sep 2022 10:42:09 +0200 Subject: [PATCH 48/83] Refactoring of quantity serialization. --- .../pathling/encoders/QuantitySupport.scala | 71 +++++++++++++++++++ .../pathling/encoders/SchemaConverter.scala | 20 +++--- .../pathling/encoders/SerializerBuilder.scala | 36 +++------- 3 files changed, 94 insertions(+), 33 deletions(-) create mode 100644 encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala new file mode 100644 index 0000000000..17e5e397da --- /dev/null +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala @@ -0,0 +1,71 @@ +/* + * This is a modified version of the Bunsen library, originally published at + * https://github.com/cerner/bunsen. + * + * Bunsen is copyright 2017 Cerner Innovation, Inc., and is licensed under + * the Apache License, version 2.0 (http://www.apache.org/licenses/LICENSE-2.0). + * + * These modifications are copyright © 2018-2022, Commonwealth Scientific + * and Industrial Research Organisation (CSIRO) ABN 41 687 119 230. Licensed + * under the CSIRO Open Source Software Licence Agreement. + * + */ + +package au.csiro.pathling.encoders + +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder +import au.csiro.pathling.encoders.terminology.ucum.Ucum +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} +import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType, StructField} +import org.apache.spark.unsafe.types.UTF8String + +/** + * Helper class for serialization of Quantities. In order to improve the performance of + * comparisons we store with each quantity canonicalized value (its value in the base unit of the quantity) + * and the base (canonicalized) unit code. + */ +object QuantitySupport { + + val VALUE_CANONICALIZED_FIELD_NAME = "_value_canonicalized" + val CODE_CANONICALIZED_FIELD_NAME = "_code_canonicalized" + + /** + * Creates additional serializers for serialization of quantities. + * + * @param expression the Quantity expression to serialize + * @return the serializers for canonicalized fields. + */ + def createExtraSerializers(expression: Expression): Seq[(String, Expression)] = { + val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) + val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) + val canonicalizedValue = StaticInvoke(classOf[Decimal], + DecimalCustomCoder.decimalType, + "apply", + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), + "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) + val canonicalizedCode = + StaticInvoke( + classOf[UTF8String], + DataTypes.StringType, + "fromString", + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), "getCanonicalCode", + Seq(valueExp, codeExp)) :: Nil) + Seq( + (VALUE_CANONICALIZED_FIELD_NAME, canonicalizedValue), + (CODE_CANONICALIZED_FIELD_NAME, canonicalizedCode) + ) + } + + /** + * Creates additional schema fields for quantities. + * + * @return the schema of canonicalized fields. + */ + def createExtraSchemaFields(): Seq[StructField] = { + Seq( + StructField(VALUE_CANONICALIZED_FIELD_NAME, DecimalCustomCoder.decimalType), + StructField(CODE_CANONICALIZED_FIELD_NAME, DataTypes.StringType) + ) + } +} diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala index 44b83e5f44..4ac654f317 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SchemaConverter.scala @@ -14,6 +14,7 @@ package au.csiro.pathling.encoders import au.csiro.pathling.encoders.ExtensionSupport.{EXTENSIONS_FIELD_NAME, FID_FIELD_NAME} +import au.csiro.pathling.encoders.QuantitySupport.{CODE_CANONICALIZED_FIELD_NAME, VALUE_CANONICALIZED_FIELD_NAME} import au.csiro.pathling.encoders.datatypes.{DataTypeMappings, DecimalCustomCoder} import au.csiro.pathling.schema.SchemaVisitor import au.csiro.pathling.schema.SchemaVisitor.isCollection @@ -54,16 +55,19 @@ private[encoders] class SchemaConverterProcessor(override val fhirContext: FhirC } } + private def createQuantityFields(definition: BaseRuntimeElementCompositeDefinition[_]): Seq[StructField] = { + definition.getImplementingClass match { + case cls if classOf[Quantity].isAssignableFrom(cls) => QuantitySupport + .createExtraSchemaFields() + case _ => Nil + } + } + override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[StructField]): DataType = { - val updatedFields = definition.getImplementingClass match { - case cls if classOf[Quantity].isAssignableFrom(cls) => fields ++ Seq( - StructField("_value_canonicalized", DecimalCustomCoder.decimalType), - StructField("_code_canonicalized", DataTypes.StringType) - ) - case _ => fields - } - StructType(updatedFields ++ createFidField() ++ createExtensionField(definition)) + StructType( + fields ++ createQuantityFields(definition) ++ createFidField() ++ createExtensionField( + definition)) } override def buildElement(elementName: String, elementValue: DataType, diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala index 903e04f6c4..69def3d53a 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/SerializerBuilder.scala @@ -14,6 +14,7 @@ package au.csiro.pathling.encoders import au.csiro.pathling.encoders.ExtensionSupport.{EXTENSIONS_FIELD_NAME, FID_FIELD_NAME} +import au.csiro.pathling.encoders.QuantitySupport.{CODE_CANONICALIZED_FIELD_NAME, VALUE_CANONICALIZED_FIELD_NAME} import au.csiro.pathling.encoders.SerializerBuilderProcessor.{dataTypeToUtf8Expr, getChildExpression, objectTypeFor} import au.csiro.pathling.encoders.datatypes.{DataTypeMappings, DecimalCustomCoder} import au.csiro.pathling.encoders.terminology.ucum.Ucum @@ -107,35 +108,20 @@ private[encoders] class SerializerBuilderProcessor(expression: Expression, expression :: Nil)) :: maybeExtensionValueField } - override def proceedCompositeChildren(value: CompositeCtx[Expression, (String, Expression)]): Seq[(String, Expression)] = { - + private def createSyntheticSerializers(value: CompositeCtx[Expression, (String, Expression)]): Seq[(String, Expression)] = { value.compositeDefinition.getImplementingClass match { - case cls if classOf[Quantity].isAssignableFrom(cls) => - val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) - val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) - // TODO: Maybe create specialized UCUM functions returning spark.Decimal and UTF8String - // TODO: Maybe move to overrideCompositeExpression() providing it with a callback to generate default fields on request (a lazy call to super) - val canonicalizedValue = StaticInvoke(classOf[Decimal], - DecimalCustomCoder.decimalType, - "apply", - StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), - "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) - val canonicalizedCode = - StaticInvoke( - classOf[UTF8String], - DataTypes.StringType, - "fromString", - StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), "getCanonicalCode", - Seq(valueExp, codeExp)) :: Nil) - super.proceedCompositeChildren(value) ++ Seq( - ("_value_canonicalized", canonicalizedValue), - ("_code_canonicalized", canonicalizedCode) - ) - case _ => dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition) - .getOrElse(super.proceedCompositeChildren(value)) + case cls if classOf[Quantity].isAssignableFrom(cls) => QuantitySupport + .createExtraSerializers(expression) + case _ => Nil } } + override def proceedCompositeChildren(value: CompositeCtx[Expression, (String, Expression)]): Seq[(String, Expression)] = { + dataTypeMappings.overrideCompositeExpression(expression, value.compositeDefinition) + .getOrElse(super.proceedCompositeChildren(value) ++ createSyntheticSerializers(value)) + + } + override def buildComposite(definition: BaseRuntimeElementCompositeDefinition[_], fields: Seq[(String, Expression)]): Expression = { From 7171fe633525707c9d62beba492c96f1ecdb21c7 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 5 Sep 2022 11:01:10 +0200 Subject: [PATCH 49/83] Fixing CodeFactor formatting issues. --- .../main/scala/au/csiro/pathling/encoders/QuantitySupport.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala index 17e5e397da..99a1ac61d5 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType, StructField} import org.apache.spark.unsafe.types.UTF8String /** - * Helper class for serialization of Quantities. In order to improve the performance of + * Helper class for serialization of Quantities. In order to improve the performance of * comparisons we store with each quantity canonicalized value (its value in the base unit of the quantity) * and the base (canonicalized) unit code. */ From d74c98da3e43db90bd0124116281714fc9ebb9ce Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 5 Sep 2022 11:24:11 +0200 Subject: [PATCH 50/83] Refactoring for Quantity cannocalisation tests. --- .../encoders/LightweightFhirEncodersTest.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index bbe61fbd18..562e96553d 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -98,6 +98,17 @@ public void assertSingletNestedExtension(final String expectedUrl, nestedConsumer.accept((Row) nestedExtensions.apply(0)); } + private static void assertQuantity(final Row quantityRow, final String canonicalizedValue, + final String canonicalizedCode) { + final BigDecimal actualCanonicalizedValue = quantityRow.getDecimal( + quantityRow.fieldIndex(QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME())); + final String actualCanonicalizedCode = quantityRow.getString( + quantityRow.fieldIndex(QuantitySupport.CODE_CANONICALIZED_FIELD_NAME())); + + assertEquals(new BigDecimal(canonicalizedValue), actualCanonicalizedValue); + assertEquals(canonicalizedCode, actualCanonicalizedCode); + } + @Test public void testDecimalCollection() { final ExpressionEncoder encoder = fhirEncoders @@ -227,7 +238,7 @@ public void testQuantityCanonicalization() { } @Test - public void testQuantityArray() { + public void testQuantityArrayCanonicalization() { final ExpressionEncoder encoder = fhirEncoders.of(Device.class); final Device device = TestData.newDevice(); @@ -250,13 +261,4 @@ public void testQuantityArray() { assertQuantity(quantity2, "0.002000", "m"); } - private void assertQuantity(final Row quantity2, final String value, final String unit) { - final BigDecimal canonicalizedValue2 = quantity2.getDecimal( - quantity2.fieldIndex("_value_canonicalized")); - final String canonicalizedCode2 = quantity2.getString( - quantity2.fieldIndex("_code_canonicalized")); - - assertEquals(new BigDecimal(value), canonicalizedValue2); - assertEquals(unit, canonicalizedCode2); - } } From 9b4fccb6a831ba73542af5e27cfa92a253678528 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 5 Sep 2022 20:13:26 +0200 Subject: [PATCH 51/83] Refactoring comparison implementation to instances od Comparable. --- .../csiro/pathling/fhirpath/Comparable.java | 130 ++++++++++++++---- .../fhirpath/comparison/CodingComparator.java | 50 +++++++ .../comparison/QuantityComparator.java | 79 +++++++++++ .../fhirpath/element/BooleanPath.java | 2 +- .../pathling/fhirpath/element/CodingPath.java | 40 +----- .../fhirpath/element/DecimalPath.java | 2 +- .../fhirpath/element/IntegerPath.java | 2 +- .../fhirpath/element/QuantityPath.java | 26 +--- .../pathling/fhirpath/element/StringPath.java | 2 +- .../pathling/fhirpath/element/TimePath.java | 2 +- .../fhirpath/literal/BooleanLiteralPath.java | 2 +- .../fhirpath/literal/DecimalLiteralPath.java | 2 +- .../fhirpath/literal/IntegerLiteralPath.java | 2 +- .../fhirpath/literal/StringLiteralPath.java | 2 +- .../fhirpath/literal/TimeLiteralPath.java | 2 +- 15 files changed, 255 insertions(+), 90 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java index 4c5653d282..281cd5ed7e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java @@ -6,26 +6,91 @@ package au.csiro.pathling.fhirpath; -import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; +import org.apache.commons.lang3.function.TriFunction; import org.apache.spark.sql.Column; +import org.apache.spark.sql.functions; /** * Describes a path that can be compared with other paths, e.g. for equality. * * @author John Grimes + * @author Piotr Szul */ public interface Comparable { + /** + * The inteface that defines comparison operation on columns. The actual implemenation and the + * implemented operation depend on the type of value in the column. + */ + public interface Comparator { + + Column equalsTo(Column left, Column right); + + default Column notEqual(Column left, Column right) { + return functions.not(equalsTo(left, right)); + } + + Column lessThan(Column left, Column right); + + default Column lessThanOrEqual(final Column left, final Column right) { + return lessThan(left, right).or(equalsTo(left, right)); + } + + Column greaterThan(Column left, Column right); + + default Column greaterThanOrEqual(final Column left, final Column right) { + return greaterThan(left, right).or(equalsTo(left, right)); + } + } + + /** + * The implementation of comparator that use the standard Spark SQL operators. + */ + class StandardComparator implements Comparator { + + @Override + public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { + return left.equalTo(right); + } + + @Override + public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { + return left.notEqual(right); + } + + @Override + public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { + return left.lt(right); + } + + @Override + public Column lessThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { + return left.leq(right); + } + + @Override + public Column greaterThan(@Nonnull final Column left, @Nonnull final Column right) { + return left.gt(right); + } + + @Override + public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { + return left.geq(right); + } + } + + Comparator STD_COMPARATOR = new StandardComparator(); + /** * Get a function that can take two Comparable paths and return a {@link Column} that contains a - * comparison condition. The type of condition is controlled by supplying a - * {@link ComparisonOperation}. + * comparison condition. The type of condition is controlled by supplying a {@link + * ComparisonOperation}. * * @param operation The {@link ComparisonOperation} type to retrieve a comparison for - * @return A {@link Function} that takes a Comparable as its parameter, and returns a - * {@link Column} + * @return A {@link Function} that takes a Comparable as its parameter, and returns a {@link + * Column} */ @Nonnull Function getComparison(@Nonnull ComparisonOperation operation); @@ -45,16 +110,35 @@ public interface Comparable { boolean isComparableTo(@Nonnull Class type); /** - * Builds a comparison function for directly comparable paths. + * Builds a comparison function for directly comparable paths using the custom comparator. * * @param source The path to build the comparison function for - * @param sparkFunction The Spark column function to use + * @param operation The {@link ComparisonOperation} type to retrieve a comparison for + * @param comparator The {@link Comparator} to use * @return A new {@link Function} */ @Nonnull static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final BiFunction sparkFunction) { - return target -> sparkFunction.apply(source.getValueColumn(), target.getValueColumn()); + @Nonnull final ComparisonOperation operation, @Nonnull final Comparator comparator) { + + final TriFunction compFunction = operation.compFunction; + + return target -> compFunction + .apply(comparator, source.getValueColumn(), target.getValueColumn()); + } + + /** + * Builds a comparison function for directly comparable paths using the standard Spark SQL + * comparison operators. + * + * @param source The path to build the comparison function for + * @param operation The {@link ComparisonOperation} type to retrieve a comparison for + * @return A new {@link Function} + */ + static Function buildComparison(@Nonnull final Comparable source, + @Nonnull final ComparisonOperation operation) { + + return buildComparison(source, operation, STD_COMPARATOR); } /** @@ -64,52 +148,43 @@ enum ComparisonOperation { /** * The equals operation. */ - EQUALS("=", Column::equalTo), + EQUALS("=", Comparator::equalsTo), /** * The not equals operation. */ - NOT_EQUALS("!=", Column::notEqual), + NOT_EQUALS("!=", Comparator::notEqual), /** * The less than or equal to operation. */ - LESS_THAN_OR_EQUAL_TO("<=", Column::leq), + LESS_THAN_OR_EQUAL_TO("<=", Comparator::lessThanOrEqual), /** * The less than operation. */ - LESS_THAN("<", Column::lt), + LESS_THAN("<", Comparator::lessThan), /** * The greater than or equal to operation. */ - GREATER_THAN_OR_EQUAL_TO(">=", Column::geq), + GREATER_THAN_OR_EQUAL_TO(">=", Comparator::greaterThanOrEqual), /** * The greater than operation. */ - GREATER_THAN(">", Column::gt); + GREATER_THAN(">", Comparator::greaterThan); @Nonnull private final String fhirPath; - /** - * A Spark function that can be used to execute this type of comparison for simple types. - * Complex types such as Coding and Quantity will implement their own comparison functions. - */ @Nonnull - private final BiFunction sparkFunction; + private final TriFunction compFunction; ComparisonOperation(@Nonnull final String fhirPath, - @Nonnull final BiFunction sparkFunction) { + @Nonnull final TriFunction compFunction) { this.fhirPath = fhirPath; - this.sparkFunction = sparkFunction; - } - - @Nonnull - public BiFunction getSparkFunction() { - return sparkFunction; + this.compFunction = compFunction; } @Override @@ -117,6 +192,5 @@ public BiFunction getSparkFunction() { public String toString() { return fhirPath; } - } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java new file mode 100644 index 0000000000..dd4f40700a --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.comparison; + +import au.csiro.pathling.errors.InvalidUserInputError; +import au.csiro.pathling.fhirpath.Comparable.Comparator; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.functions; +import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.List; + +import static org.apache.spark.sql.functions.lit; + +/** + * Implementation of comparator for Coding type. + */ +public class CodingComparator implements Comparator { + + private static final List EQUALITY_COLUMNS = Arrays + .asList("system", "code", "version", "display", "userSelected"); + + public static final CodingComparator INSTANCE = new CodingComparator(); + + @Override + public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { + return functions.when(left.isNull().or(right.isNull()), lit(null)) + .otherwise( + EQUALITY_COLUMNS.stream() + .map(f -> left.getField(f).eqNullSafe(right.getField(f))).reduce(Column::and).get() + ); + } + + @Override + public Column lessThan(final Column left, final Column right) { + throw new InvalidUserInputError( + "Coding type does not support comparison operator: " + "lessThan"); + + } + + @Override + public Column greaterThan(final Column left, final Column right) { + throw new InvalidUserInputError( + "Coding type does not support comparison operator: " + "greaterThan"); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java new file mode 100644 index 0000000000..e2fe404d72 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.comparison; + +import au.csiro.pathling.fhirpath.Comparable.Comparator; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import org.apache.spark.sql.Column; +import javax.annotation.Nonnull; +import java.util.function.BiFunction; + +import static au.csiro.pathling.fhirpath.Comparable.STD_COMPARATOR; +import static org.apache.spark.sql.functions.when; + +/** + * Implementation of comparator for the Quantity type. It uses canonicalized values and units for + * comparison rather than the original values. + */ +public class QuantityComparator implements Comparator { + + public final static QuantityComparator INSTANCE = new QuantityComparator(STD_COMPARATOR); + + private final Comparator defaultComparator; + + public QuantityComparator(final Comparator defaultComparator) { + this.defaultComparator = defaultComparator; + } + + private static BiFunction wrap( + @Nonnull final BiFunction function) { + + return (left, right) -> { + final Column sourceCode = left.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column targetCode = right.getField( + QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column sourceValue = left.getField( + QuantityEncoding.CANONICALIZED_VALUE_COLUMN); + final Column targetValue = right.getField( + QuantityEncoding.CANONICALIZED_VALUE_COLUMN); + return when(sourceCode.equalTo(targetCode), + function.apply(sourceValue, targetValue)).otherwise( + null); + }; + } + + @Override + public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::equalsTo).apply(left, right); + } + + @Override + public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::notEqual).apply(left, right); + } + + @Override + public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::lessThan).apply(left, right); + } + + @Override + public Column lessThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::lessThanOrEqual).apply(left, right); + } + + @Override + public Column greaterThan(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::greaterThan).apply(left, right); + } + + @Override + public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { + return wrap(defaultComparator::greaterThanOrEqual).apply(left, right); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/BooleanPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/BooleanPath.java index 020993e42d..68ea58921a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/BooleanPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/BooleanPath.java @@ -70,7 +70,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java index 18003e7b1d..c1d0f2e037 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java @@ -14,6 +14,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.comparison.CodingComparator; import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.terminology.CodingToLiteral; @@ -38,10 +39,6 @@ */ public class CodingPath extends ElementPath implements Materializable, Comparable { - private static final List EQUALITY_COLUMNS = Arrays - .asList("system", "code", "version", "display", "userSelected"); - - private static final ImmutableSet> COMPARABLE_TYPES = ImmutableSet .of(CodingPath.class, CodingLiteralPath.class, NullLiteralPath.class); @@ -92,6 +89,11 @@ public static Optional valueFromRow(@Nonnull final Row row, final int co return Optional.of(coding); } + @Nonnull + public static ImmutableSet> getComparableTypes() { + return COMPARABLE_TYPES; + } + /** * Builds a comparison function for Coding paths. * @@ -103,35 +105,7 @@ public static Optional valueFromRow(@Nonnull final Row row, final int co @Nonnull public static Function buildComparison(@Nonnull final Comparable source, @Nonnull final ComparisonOperation operation) { - if (ComparisonOperation.EQUALS.equals(operation)) { - return Comparable.buildComparison(source, codingEqual()); - } else if (ComparisonOperation.NOT_EQUALS.equals(operation)) { - return Comparable.buildComparison(source, codingNotEqual()); - } else { - throw new InvalidUserInputError( - "Coding type does not support comparison operator: " + operation); - } - } - - @Nonnull - private static BiFunction codingEqual() { - //noinspection OptionalGetWithoutIsPresent - return (l, r) -> - functions.when(l.isNull().or(r.isNull()), lit(null)) - .otherwise( - EQUALITY_COLUMNS.stream() - .map(f -> l.getField(f).eqNullSafe(r.getField(f))).reduce(Column::and).get() - ); - } - - @Nonnull - private static BiFunction codingNotEqual() { - return codingEqual().andThen(functions::not); - } - - @Nonnull - public static ImmutableSet> getComparableTypes() { - return COMPARABLE_TYPES; + return Comparable.buildComparison(source, operation, CodingComparator.INSTANCE); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java index 2a334a8275..b3c41cfdf2 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DecimalPath.java @@ -91,7 +91,7 @@ public static Optional valueFromRow(@Nonnull final Row row, final i @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } public static org.apache.spark.sql.types.DecimalType getDecimalType() { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java index 9dc3ccfedb..f27fa48434 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/IntegerPath.java @@ -106,7 +106,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index fb86c88711..c4164942f0 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -15,6 +15,7 @@ import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.ResourcePath; +import au.csiro.pathling.fhirpath.comparison.QuantityComparator; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; @@ -47,28 +48,15 @@ protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset< } @Nonnull - @Override - public Function getComparison(@Nonnull final ComparisonOperation operation) { - return buildComparison(this, operation); + public static Function buildComparison(@Nonnull final Comparable source, + @Nonnull final ComparisonOperation operation) { + return Comparable.buildComparison(source, operation, QuantityComparator.INSTANCE); } @Nonnull - public static Function buildComparison(@Nonnull final FhirPath source, - @Nonnull final ComparisonOperation operation) { - return target -> { - final Column comparableSource = source.getValueColumn(); - final Column comparableTarget = target.getValueColumn(); - final Column sourceCode = comparableSource.getField( - QuantityEncoding.CANONICALIZED_CODE_COLUMN); - final Column targetCode = comparableTarget.getField( - QuantityEncoding.CANONICALIZED_CODE_COLUMN); - final Column sourceValue = comparableSource.getField( - QuantityEncoding.CANONICALIZED_VALUE_COLUMN); - final Column targetValue = comparableTarget.getField( - QuantityEncoding.CANONICALIZED_VALUE_COLUMN); - final Column compareValues = operation.getSparkFunction().apply(sourceValue, targetValue); - return when(sourceCode.equalTo(targetCode), compareValues).otherwise(null); - }; + @Override + public Function getComparison(@Nonnull final ComparisonOperation operation) { + return buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/StringPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/StringPath.java index 9acb81ed8d..73b484c39e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/StringPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/StringPath.java @@ -98,7 +98,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java index c2f4e25952..eedf4bb267 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/TimePath.java @@ -70,7 +70,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java index 4e8a62edca..a3b23cf61b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/BooleanLiteralPath.java @@ -65,7 +65,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java index b7244d55d8..4d7541916b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DecimalLiteralPath.java @@ -82,7 +82,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java index 730519b396..984fffac20 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/IntegerLiteralPath.java @@ -70,7 +70,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java index 01ebe1c999..6dbbae6d12 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/StringLiteralPath.java @@ -93,7 +93,7 @@ public static String unescapeFhirPathString(@Nonnull String value) { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java index e4a77b18f9..2f6373c699 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/TimeLiteralPath.java @@ -69,7 +69,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(this, operation.getSparkFunction()); + return Comparable.buildComparison(this, operation); } @Override From 76cbd6f8faacfe8d00a92a70077068b95bbc40ff Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 6 Sep 2022 10:48:24 +0200 Subject: [PATCH 52/83] Refactoring comparison implementation for DateTime types. --- .../csiro/pathling/fhirpath/Comparable.java | 34 +++++------ ...mparator.java => CodingSqlComparator.java} | 24 +++++++- .../comparison/DateTimeSqlComparator.java | 61 +++++++++++++++++++ ...arator.java => QuantitySqlComparator.java} | 43 +++++++++---- .../pathling/fhirpath/element/CodingPath.java | 26 +------- .../pathling/fhirpath/element/DatePath.java | 3 +- .../fhirpath/element/DateTimePath.java | 49 +-------------- .../fhirpath/element/QuantityPath.java | 11 +--- .../fhirpath/literal/CodingLiteralPath.java | 3 +- .../fhirpath/literal/DateLiteralPath.java | 3 +- .../fhirpath/literal/DateTimeLiteralPath.java | 3 +- .../fhirpath/literal/QuantityLiteralPath.java | 3 +- site/docs/fhirpath/operators.md | 4 +- 13 files changed, 148 insertions(+), 119 deletions(-) rename fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/{CodingComparator.java => CodingSqlComparator.java} (61%) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java rename fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/{QuantityComparator.java => QuantitySqlComparator.java} (56%) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java index 281cd5ed7e..791fcb9060 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java @@ -21,10 +21,10 @@ public interface Comparable { /** - * The inteface that defines comparison operation on columns. The actual implemenation and the + * The interface that defines comparison operation on columns. The actual implementation and the * implemented operation depend on the type of value in the column. */ - public interface Comparator { + interface SqlComparator { Column equalsTo(Column left, Column right); @@ -48,7 +48,7 @@ default Column greaterThanOrEqual(final Column left, final Column right) { /** * The implementation of comparator that use the standard Spark SQL operators. */ - class StandardComparator implements Comparator { + class StandardSqlComparator implements SqlComparator { @Override public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { @@ -81,7 +81,7 @@ public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Colu } } - Comparator STD_COMPARATOR = new StandardComparator(); + SqlComparator STD_SQL_COMPARATOR = new StandardSqlComparator(); /** * Get a function that can take two Comparable paths and return a {@link Column} that contains a @@ -114,17 +114,17 @@ public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Colu * * @param source The path to build the comparison function for * @param operation The {@link ComparisonOperation} type to retrieve a comparison for - * @param comparator The {@link Comparator} to use + * @param sqlComparator The {@link SqlComparator} to use * @return A new {@link Function} */ @Nonnull static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final ComparisonOperation operation, @Nonnull final Comparator comparator) { + @Nonnull final ComparisonOperation operation, @Nonnull final SqlComparator sqlComparator) { - final TriFunction compFunction = operation.compFunction; + final TriFunction compFunction = operation.compFunction; return target -> compFunction - .apply(comparator, source.getValueColumn(), target.getValueColumn()); + .apply(sqlComparator, source.getValueColumn(), target.getValueColumn()); } /** @@ -138,7 +138,7 @@ static Function buildComparison(@Nonnull final Comparable so static Function buildComparison(@Nonnull final Comparable source, @Nonnull final ComparisonOperation operation) { - return buildComparison(source, operation, STD_COMPARATOR); + return buildComparison(source, operation, STD_SQL_COMPARATOR); } /** @@ -148,41 +148,41 @@ enum ComparisonOperation { /** * The equals operation. */ - EQUALS("=", Comparator::equalsTo), + EQUALS("=", SqlComparator::equalsTo), /** * The not equals operation. */ - NOT_EQUALS("!=", Comparator::notEqual), + NOT_EQUALS("!=", SqlComparator::notEqual), /** * The less than or equal to operation. */ - LESS_THAN_OR_EQUAL_TO("<=", Comparator::lessThanOrEqual), + LESS_THAN_OR_EQUAL_TO("<=", SqlComparator::lessThanOrEqual), /** * The less than operation. */ - LESS_THAN("<", Comparator::lessThan), + LESS_THAN("<", SqlComparator::lessThan), /** * The greater than or equal to operation. */ - GREATER_THAN_OR_EQUAL_TO(">=", Comparator::greaterThanOrEqual), + GREATER_THAN_OR_EQUAL_TO(">=", SqlComparator::greaterThanOrEqual), /** * The greater than operation. */ - GREATER_THAN(">", Comparator::greaterThan); + GREATER_THAN(">", SqlComparator::greaterThan); @Nonnull private final String fhirPath; @Nonnull - private final TriFunction compFunction; + private final TriFunction compFunction; ComparisonOperation(@Nonnull final String fhirPath, - @Nonnull final TriFunction compFunction) { + @Nonnull final TriFunction compFunction) { this.fhirPath = fhirPath; this.compFunction = compFunction; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java similarity index 61% rename from fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java rename to fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java index dd4f40700a..5d1b4dc2db 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java @@ -7,27 +7,31 @@ package au.csiro.pathling.fhirpath.comparison; import au.csiro.pathling.errors.InvalidUserInputError; -import au.csiro.pathling.fhirpath.Comparable.Comparator; +import au.csiro.pathling.fhirpath.Comparable; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; +import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; import org.apache.spark.sql.Column; import org.apache.spark.sql.functions; import javax.annotation.Nonnull; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import static org.apache.spark.sql.functions.lit; /** * Implementation of comparator for Coding type. */ -public class CodingComparator implements Comparator { +public class CodingSqlComparator implements SqlComparator { private static final List EQUALITY_COLUMNS = Arrays .asList("system", "code", "version", "display", "userSelected"); - public static final CodingComparator INSTANCE = new CodingComparator(); + private static final CodingSqlComparator INSTANCE = new CodingSqlComparator(); @Override public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { + //noinspection OptionalGetWithoutIsPresent return functions.when(left.isNull().or(right.isNull()), lit(null)) .otherwise( EQUALITY_COLUMNS.stream() @@ -47,4 +51,18 @@ public Column greaterThan(final Column left, final Column right) { throw new InvalidUserInputError( "Coding type does not support comparison operator: " + "greaterThan"); } + + /** + * Builds a comparison function for Coding paths. + * + * @param source The path to build the comparison function for + * @param operation The {@link au.csiro.pathling.fhirpath.Comparable.ComparisonOperation} type to + * build + * @return A new {@link Function} + */ + @Nonnull + public static Function buildComparison(@Nonnull final Comparable source, + @Nonnull final ComparisonOperation operation) { + return Comparable.buildComparison(source, operation, INSTANCE); + } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java new file mode 100644 index 0000000000..3884847571 --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java @@ -0,0 +1,61 @@ +package au.csiro.pathling.fhirpath.comparison; + +import static org.apache.spark.sql.functions.callUDF; + +import au.csiro.pathling.fhirpath.Comparable; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; +import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; +import au.csiro.pathling.sql.dates.datetime.DateTimeEqualsFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanOrEqualToFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; +import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; +import org.apache.spark.sql.Column; +import javax.annotation.Nonnull; +import java.util.function.Function; + +/** + * Implementation of comparator for the DateTime type. + */ +public class DateTimeSqlComparator implements SqlComparator { + + private static final DateTimeSqlComparator INSTANCE = new DateTimeSqlComparator(); + + @Override + public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { + return callUDF(DateTimeEqualsFunction.FUNCTION_NAME, left, right); + } + + @Override + public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { + return callUDF(DateTimeLessThanFunction.FUNCTION_NAME, left, right); + } + + @Override + public Column lessThanOrEqual(final Column left, final Column right) { + return callUDF(DateTimeLessThanOrEqualToFunction.FUNCTION_NAME, left, right); + } + + @Override + public Column greaterThan(final Column left, final Column right) { + return callUDF(DateTimeGreaterThanFunction.FUNCTION_NAME, left, right); + } + + @Override + public Column greaterThanOrEqual(final Column left, final Column right) { + return callUDF(DateTimeGreaterThanOrEqualToFunction.FUNCTION_NAME, left, right); + } + + /** + * Builds a comparison function for date and date/time like paths. + * + * @param source the path to build the comparison function for + * @param operation the {@link ComparisonOperation} that should be built + * @return a new {@link Function} + */ + @Nonnull + public static Function buildComparison(@Nonnull final Comparable source, + @Nonnull final ComparisonOperation operation) { + return Comparable.buildComparison(source, operation, INSTANCE); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java similarity index 56% rename from fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java rename to fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java index e2fe404d72..2b894d6a4a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantityComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java @@ -6,27 +6,31 @@ package au.csiro.pathling.fhirpath.comparison; -import au.csiro.pathling.fhirpath.Comparable.Comparator; +import au.csiro.pathling.fhirpath.Comparable; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; +import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import org.apache.spark.sql.Column; import javax.annotation.Nonnull; import java.util.function.BiFunction; +import java.util.function.Function; -import static au.csiro.pathling.fhirpath.Comparable.STD_COMPARATOR; +import static au.csiro.pathling.fhirpath.Comparable.STD_SQL_COMPARATOR; import static org.apache.spark.sql.functions.when; /** * Implementation of comparator for the Quantity type. It uses canonicalized values and units for * comparison rather than the original values. */ -public class QuantityComparator implements Comparator { +public class QuantitySqlComparator implements SqlComparator { - public final static QuantityComparator INSTANCE = new QuantityComparator(STD_COMPARATOR); + private final static QuantitySqlComparator INSTANCE = new QuantitySqlComparator( + STD_SQL_COMPARATOR); - private final Comparator defaultComparator; + private final SqlComparator defaultSqlComparator; - public QuantityComparator(final Comparator defaultComparator) { - this.defaultComparator = defaultComparator; + public QuantitySqlComparator(final SqlComparator defaultSqlComparator) { + this.defaultSqlComparator = defaultSqlComparator; } private static BiFunction wrap( @@ -49,31 +53,44 @@ private static BiFunction wrap( @Override public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::equalsTo).apply(left, right); + return wrap(defaultSqlComparator::equalsTo).apply(left, right); } @Override public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::notEqual).apply(left, right); + return wrap(defaultSqlComparator::notEqual).apply(left, right); } @Override public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::lessThan).apply(left, right); + return wrap(defaultSqlComparator::lessThan).apply(left, right); } @Override public Column lessThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::lessThanOrEqual).apply(left, right); + return wrap(defaultSqlComparator::lessThanOrEqual).apply(left, right); } @Override public Column greaterThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::greaterThan).apply(left, right); + return wrap(defaultSqlComparator::greaterThan).apply(left, right); } @Override public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultComparator::greaterThanOrEqual).apply(left, right); + return wrap(defaultSqlComparator::greaterThanOrEqual).apply(left, right); + } + + /** + * Builds a comparison function for quantity like paths. + * + * @param source the path to build the comparison function for + * @param operation the {@link ComparisonOperation} that should be built + * @return a new {@link Function} + */ + @Nonnull + public static Function buildComparison(@Nonnull final Comparable source, + @Nonnull final ComparisonOperation operation) { + return Comparable.buildComparison(source, operation, INSTANCE); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java index c1d0f2e037..ccad81b639 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/CodingPath.java @@ -7,28 +7,22 @@ package au.csiro.pathling.fhirpath.element; import static org.apache.spark.sql.functions.callUDF; -import static org.apache.spark.sql.functions.lit; -import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; import au.csiro.pathling.fhirpath.ResourcePath; -import au.csiro.pathling.fhirpath.comparison.CodingComparator; +import au.csiro.pathling.fhirpath.comparison.CodingSqlComparator; import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.terminology.CodingToLiteral; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; -import java.util.List; import java.util.Optional; -import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; -import org.apache.spark.sql.functions; import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; @@ -93,25 +87,11 @@ public static Optional valueFromRow(@Nonnull final Row row, final int co public static ImmutableSet> getComparableTypes() { return COMPARABLE_TYPES; } - - /** - * Builds a comparison function for Coding paths. - * - * @param source The path to build the comparison function for - * @param operation The {@link au.csiro.pathling.fhirpath.Comparable.ComparisonOperation} type to - * build - * @return A new {@link Function} - */ - @Nonnull - public static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(source, operation, CodingComparator.INSTANCE); - } - + @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return buildComparison(this, operation); + return CodingSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java index 999dc8b1dc..30e8898ade 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DatePath.java @@ -15,6 +15,7 @@ import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.comparison.DateTimeSqlComparator; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; @@ -89,7 +90,7 @@ public static Optional valueFromRow(@Nonnull final Row row, final int @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation); + return DateTimeSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java index b7c34d395e..b87b23100c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/DateTimePath.java @@ -8,7 +8,6 @@ import static au.csiro.pathling.fhirpath.Temporal.buildDateArithmeticOperation; import static org.apache.spark.sql.functions.callUDF; -import static org.apache.spark.sql.functions.not; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; @@ -16,16 +15,12 @@ import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.comparison.DateTimeSqlComparator; import au.csiro.pathling.fhirpath.literal.DateLiteralPath; import au.csiro.pathling.fhirpath.literal.DateTimeLiteralPath; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; -import au.csiro.pathling.sql.dates.datetime.DateTimeEqualsFunction; -import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanFunction; -import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanOrEqualToFunction; -import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; -import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; import com.google.common.collect.ImmutableSet; import java.util.Optional; @@ -90,46 +85,6 @@ public static Optional valueFromRow(@Nonnull final Row row, } } - /** - * Builds a comparison function for date and date/time like paths. - * - * @param source the path to build the comparison function for - * @param operation the {@link ComparisonOperation} that should be built - * @return a new {@link Function} - */ - @Nonnull - public static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final ComparisonOperation operation) { - return (target) -> { - final String functionName; - switch (operation) { - case EQUALS: - case NOT_EQUALS: - functionName = DateTimeEqualsFunction.FUNCTION_NAME; - final Column equals = callUDF(functionName, - source.getValueColumn(), target.getValueColumn()); - return operation == ComparisonOperation.EQUALS - ? equals - : not(equals); - case LESS_THAN: - functionName = DateTimeLessThanFunction.FUNCTION_NAME; - break; - case LESS_THAN_OR_EQUAL_TO: - functionName = DateTimeLessThanOrEqualToFunction.FUNCTION_NAME; - break; - case GREATER_THAN: - functionName = DateTimeGreaterThanFunction.FUNCTION_NAME; - break; - case GREATER_THAN_OR_EQUAL_TO: - functionName = DateTimeGreaterThanOrEqualToFunction.FUNCTION_NAME; - break; - default: - throw new AssertionError("Unsupported operation: " + operation); - } - return callUDF(functionName, source.getValueColumn(), target.getValueColumn()); - }; - } - @Nonnull public static ImmutableSet> getComparableTypes() { return COMPARABLE_TYPES; @@ -138,7 +93,7 @@ public static ImmutableSet> getComparableTypes() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return buildComparison(this, operation); + return DateTimeSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index c4164942f0..2ae130de92 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -11,11 +11,10 @@ import static org.apache.spark.sql.functions.when; import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.ResourcePath; -import au.csiro.pathling.fhirpath.comparison.QuantityComparator; +import au.csiro.pathling.fhirpath.comparison.QuantitySqlComparator; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; @@ -47,16 +46,10 @@ protected QuantityPath(@Nonnull final String expression, @Nonnull final Dataset< thisColumn, fhirType); } - @Nonnull - public static Function buildComparison(@Nonnull final Comparable source, - @Nonnull final ComparisonOperation operation) { - return Comparable.buildComparison(source, operation, QuantityComparator.INSTANCE); - } - @Nonnull @Override public Function getComparison(@Nonnull final ComparisonOperation operation) { - return buildComparison(this, operation); + return QuantitySqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java index beddfd7096..88d4bd5bef 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/CodingLiteralPath.java @@ -12,6 +12,7 @@ import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Materializable; +import au.csiro.pathling.fhirpath.comparison.CodingSqlComparator; import au.csiro.pathling.fhirpath.element.CodingPath; import java.util.Optional; import java.util.function.Function; @@ -83,7 +84,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return CodingPath.buildComparison(this, operation); + return CodingSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java index 6b508ab3d2..21fa74f686 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateLiteralPath.java @@ -14,6 +14,7 @@ import au.csiro.pathling.fhirpath.Materializable; import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.comparison.DateTimeSqlComparator; import au.csiro.pathling.fhirpath.element.DatePath; import au.csiro.pathling.fhirpath.element.DateTimePath; import au.csiro.pathling.sql.dates.date.DateAddDurationFunction; @@ -76,7 +77,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation); + return DateTimeSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java index 0fa1dc1a1a..2d607447fe 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/DateTimeLiteralPath.java @@ -14,6 +14,7 @@ import au.csiro.pathling.fhirpath.Materializable; import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.comparison.DateTimeSqlComparator; import au.csiro.pathling.fhirpath.element.DateTimePath; import au.csiro.pathling.sql.dates.datetime.DateTimeAddDurationFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeSubtractDurationFunction; @@ -79,7 +80,7 @@ public Column buildValueColumn() { @Override @Nonnull public Function getComparison(@Nonnull final ComparisonOperation operation) { - return DateTimePath.buildComparison(this, operation); + return DateTimeSqlComparator.buildComparison(this, operation); } @Override diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index e5afd5d607..eea1fb268e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -15,6 +15,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; +import au.csiro.pathling.fhirpath.comparison.QuantitySqlComparator; import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import com.google.common.collect.ImmutableMap; @@ -193,7 +194,7 @@ public Column buildValueColumn() { @Nonnull @Override public Function getComparison(@Nonnull final ComparisonOperation operation) { - return QuantityPath.buildComparison(this, operation); + return QuantitySqlComparator.buildComparison(this, operation); } @Override diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index 78fb80ee69..a802250033 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -41,7 +41,7 @@ All comparison operators return a [Boolean](/docs/fhirpath/data-types#boolean) v * Not all Quantity values are comparable, it depends upon the comparability of the units. See the [FHIRPath specification](https://hl7.org/fhirpath/#comparison) for details on -how Quantity values are compared. Quantities with a `comparator` are treated as +how Quantity values are compared. Quantities with a `sqlComparator` are treated as not comparable by this implementation. See also: [Comparison](https://hl7.org/fhirpath/#comparison) @@ -63,7 +63,7 @@ Not all Quantity, Date and DateTime values can be compared for equality, it depends upon the comparability of the units within the Quantity values. See the [FHIRPath specification](https://hl7.org/fhirpath/#quantity-equality) for details on how equality works with Quantity values. Quantities with a -`comparator` are treated as not comparable by this implementation. +`sqlComparator` are treated as not comparable by this implementation. See also: [Equality](https://hl7.org/fhirpath/#equality) From 731a77b0d65ffbe8881735b50d278093af5f121b Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 6 Sep 2022 11:45:21 +0200 Subject: [PATCH 53/83] Small refactoring changes. --- .../sql/dates/TemporalArithmeticFunction.java | 52 +++++++------------ .../pathling/test/helpers/SparkHelpers.java | 23 +------- 2 files changed, 22 insertions(+), 53 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index 81351569ce..dcf5f66b48 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -25,68 +25,56 @@ import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.Quantity; -public abstract class TemporalArithmeticFunction implements +public abstract class TemporalArithmeticFunction implements SqlFunction2 { private static final long serialVersionUID = -5016153440496309996L; - static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() - .put("year", Calendar.YEAR) - .put("years", Calendar.YEAR) - .put("month", Calendar.MONTH) - .put("months", Calendar.MONTH) - .put("day", Calendar.DATE) - .put("days", Calendar.DATE) - .put("hour", Calendar.HOUR) - .put("hours", Calendar.HOUR) - .put("minute", Calendar.MINUTE) - .put("minutes", Calendar.MINUTE) - .put("second", Calendar.SECOND) - .put("seconds", Calendar.SECOND) - .put("millisecond", Calendar.MILLISECOND) - .put("milliseconds", Calendar.MILLISECOND) - .build(); + static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder().put( + "year", Calendar.YEAR).put("years", Calendar.YEAR).put("month", Calendar.MONTH) + .put("months", Calendar.MONTH).put("day", Calendar.DATE).put("days", Calendar.DATE) + .put("hour", Calendar.HOUR).put("hours", Calendar.HOUR).put("minute", Calendar.MINUTE) + .put("minutes", Calendar.MINUTE).put("second", Calendar.SECOND) + .put("seconds", Calendar.SECOND).put("millisecond", Calendar.MILLISECOND) + .put("milliseconds", Calendar.MILLISECOND).build(); @Nonnull - protected IntermediateType performAddition(@Nonnull final IntermediateType temporal, - @Nonnull final Quantity calendarDuration) { + protected T performAddition(@Nonnull final T temporal, @Nonnull final Quantity calendarDuration) { return performArithmetic(temporal, calendarDuration, false); } @Nonnull - protected IntermediateType performSubtraction(@Nonnull final IntermediateType temporal, + protected T performSubtraction(@Nonnull final T temporal, @Nonnull final Quantity calendarDuration) { return performArithmetic(temporal, calendarDuration, true); } @Nonnull - private IntermediateType performArithmetic(final @Nonnull IntermediateType temporal, - final @Nonnull Quantity calendarDuration, final boolean subtract) { + private T performArithmetic(final @Nonnull T temporal, final @Nonnull Quantity calendarDuration, + final boolean subtract) { if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { throw new IllegalArgumentException("Calendar duration must have a system of " + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); } - final int amountToAdd = calendarDuration.getValue() - .setScale(0, RoundingMode.HALF_UP) + final int amountToAdd = calendarDuration.getValue().setScale(0, RoundingMode.HALF_UP) .intValue(); final Integer temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( calendarDuration.getCode()); checkUserInput(temporalUnit != null, "Unsupported calendar duration unit: " + calendarDuration.getCode()); - @SuppressWarnings("unchecked") - final IntermediateType result = (IntermediateType) temporal.copy(); + @SuppressWarnings("unchecked") final T result = (T) temporal.copy(); result.add(temporalUnit, subtract - ? amountToAdd * -1 + ? -amountToAdd : amountToAdd); return result; } - protected abstract Function parseEncodedValue(); + protected abstract Function parseEncodedValue(); - protected abstract BiFunction getOperationFunction(); + protected abstract BiFunction getOperationFunction(); - protected abstract Function encodeResult(); + protected abstract Function encodeResult(); @Override public DataType getReturnType() { @@ -100,9 +88,9 @@ public String call(@Nullable final String temporalValue, @Nullable final Row cal if (temporalValue == null || calendarDurationRow == null) { return null; } - final IntermediateType temporal = parseEncodedValue().apply(temporalValue); + final T temporal = parseEncodedValue().apply(temporalValue); final Quantity calendarDuration = QuantityEncoding.decode(calendarDurationRow); - final IntermediateType result = getOperationFunction().apply(temporal, calendarDuration); + final T result = getOperationFunction().apply(temporal, calendarDuration); return encodeResult().apply(result); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index bd2e99df14..8b27068b6c 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -11,6 +11,7 @@ import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import au.csiro.pathling.encoders.terminology.ucum.Ucum; +import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import java.math.BigDecimal; import java.math.RoundingMode; @@ -106,27 +107,7 @@ public static StructType referenceStructType() { @Nonnull public static StructType quantityStructType() { - final Metadata metadata = new MetadataBuilder().build(); - final StructField id = new StructField("id", DataTypes.StringType, true, metadata); - final StructField value = new StructField("value", DataTypes.createDecimalType( - DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); - final StructField valueScale = new StructField("value_scale", DataTypes.IntegerType, true, - metadata); - final StructField comparator = new StructField("comparator", DataTypes.StringType, true, - metadata); - final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); - final StructField system = new StructField("system", DataTypes.StringType, true, metadata); - final StructField code = new StructField("code", DataTypes.StringType, true, metadata); - final StructField canonicalizedCode = new StructField("_code_canonicalized", - DataTypes.StringType, true, metadata); - final StructField canonicalizedValue = new StructField("_value_canonicalized", - DataTypes.createDecimalType( - DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); - final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, - metadata); - return new StructType( - new StructField[]{id, value, valueScale, comparator, unit, system, code, canonicalizedValue, - canonicalizedCode, fid}); + return QuantityEncoding.dataType(); } @Nonnull From 44865d732043571de08ac5d192fc75086338530b Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 6 Sep 2022 21:23:56 +0200 Subject: [PATCH 54/83] Refactoring: removed explicit creation of UFDs for unit-tests. --- .../main/java/au/csiro/pathling/spark/Spark.java | 12 ++++++------ .../sql/dates/TemporalDifferenceFunction.java | 2 +- .../sql/dates/date/DateAddDurationFunction.java | 2 +- .../dates/date/DateSubtractDurationFunction.java | 2 +- .../datetime/DateTimeAddDurationFunction.java | 2 +- .../sql/dates/datetime/DateTimeEqualsFunction.java | 2 +- .../datetime/DateTimeGreaterThanFunction.java | 2 +- .../DateTimeGreaterThanOrEqualToFunction.java | 2 +- .../dates/datetime/DateTimeLessThanFunction.java | 2 +- .../DateTimeLessThanOrEqualToFunction.java | 2 +- .../datetime/DateTimeSubtractDurationFunction.java | 2 +- .../sql/dates/time/TimeEqualsFunction.java | 2 +- .../sql/dates/time/TimeGreaterThanFunction.java | 2 +- .../time/TimeGreaterThanOrEqualToFunction.java | 2 +- .../sql/dates/time/TimeLessThanFunction.java | 2 +- .../dates/time/TimeLessThanOrEqualToFunction.java | 2 +- .../pathling/terminology/CodingToLiteral.java | 2 +- .../csiro/pathling/test/UnitTestDependencies.java | 14 ++++---------- 18 files changed, 26 insertions(+), 32 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index eff616bf89..cae2352139 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -56,9 +56,9 @@ public class Spark { public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener, - @Nonnull final List sqlFunction1, - @Nonnull final List sqlFunction2, - @Nonnull final List sqlFunction3) { + @Nonnull final List> sqlFunction1, + @Nonnull final List> sqlFunction2, + @Nonnull final List> sqlFunction3) { log.debug("Creating Spark session"); resolveSparkConfiguration(environment); @@ -69,13 +69,13 @@ public static SparkSession build(@Nonnull final Configuration configuration, // Configure user defined strategy and functions. SqlStrategy.setup(spark); - for (final SqlFunction1 function : sqlFunction1) { + for (final SqlFunction1 function : sqlFunction1) { spark.udf().register(function.getName(), function, function.getReturnType()); } - for (final SqlFunction2 function : sqlFunction2) { + for (final SqlFunction2 function : sqlFunction2) { spark.udf().register(function.getName(), function, function.getReturnType()); } - for (final SqlFunction3 function : sqlFunction3) { + for (final SqlFunction3 function : sqlFunction3) { spark.udf().register(function.getName(), function, function.getReturnType()); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java index a209533999..a58f85434d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TemporalDifferenceFunction implements SqlFunction3 { private static final long serialVersionUID = -7306741471632636471L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java index 8b59519127..dc7d7bf81c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateAddDurationFunction extends DateArithmeticFunction { private static final long serialVersionUID = -5029179160644275584L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java index 5eac5038b5..fee7980aab 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateSubtractDurationFunction extends DateArithmeticFunction { private static final long serialVersionUID = 5201879133976866457L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java index 2b28b401f4..47eb326255 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeAddDurationFunction extends DateTimeArithmeticFunction { private static final long serialVersionUID = 6922227603585641053L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java index 2667c8689a..5901be60a2 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeEqualsFunction extends DateTimeComparisonFunction { private static final long serialVersionUID = -8717420985056046161L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java index 3ea5108b4d..b5ff30d9e4 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java @@ -14,7 +14,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeGreaterThanFunction extends DateTimeComparisonFunction { private static final long serialVersionUID = 6648102436817402989L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java index bc554e5de7..29d3b19556 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeGreaterThanOrEqualToFunction extends DateTimeComparisonFunction { private static final long serialVersionUID = -4680194983727139029L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java index d39354380f..310c0a06eb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java @@ -14,7 +14,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeLessThanFunction extends DateTimeComparisonFunction { private static final long serialVersionUID = -4688679934965980217L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java index 449bf2e2d1..d10964a121 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeLessThanOrEqualToFunction extends DateTimeComparisonFunction { private static final long serialVersionUID = 787654631927909813L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java index 57e3883292..1e360c2b2f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class DateTimeSubtractDurationFunction extends DateTimeArithmeticFunction { private static final long serialVersionUID = -5922228168177608861L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java index 9c430205dd..0a02d46729 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TimeEqualsFunction extends TimeComparisonFunction { private static final long serialVersionUID = -6607019777915271539L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java index edd6b033db..5a05680636 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TimeGreaterThanFunction extends TimeComparisonFunction { private static final long serialVersionUID = 1785863110270729355L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java index e70ccb6a58..2dd4315f53 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TimeGreaterThanOrEqualToFunction extends TimeComparisonFunction { private static final long serialVersionUID = -8098751595303018323L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java index 29d1d46802..f59c79c9b3 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TimeLessThanFunction extends TimeComparisonFunction { private static final long serialVersionUID = 5957315745114053261L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java index 40890f5bfd..b5a529b9d0 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java @@ -12,7 +12,7 @@ import org.springframework.stereotype.Component; @Component -@Profile("core") +@Profile("core | unit-test") public class TimeLessThanOrEqualToFunction extends TimeComparisonFunction { private static final long serialVersionUID = -5929640258789711609L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java index 0bfe3a8337..50f7ba642c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java +++ b/fhir-server/src/main/java/au/csiro/pathling/terminology/CodingToLiteral.java @@ -23,7 +23,7 @@ * @author John Grimes */ @Component -@Profile("core") +@Profile("core | unit-test") public class CodingToLiteral implements SqlFunction1 { /** diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java index 790a095a68..348d270d13 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/UnitTestDependencies.java @@ -44,6 +44,7 @@ import org.fhir.ucum.UcumService; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Profile; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; @@ -60,17 +61,10 @@ class UnitTestDependencies { @Nonnull static SparkSession sparkSession(@Nonnull final Configuration configuration, @Nonnull final Environment environment, + @Nonnull final List> sqlFunction1, + @Nonnull final List> sqlFunction2, + @Nonnull final List> sqlFunction3, @Nonnull final Optional sparkListener) { - final List sqlFunction1 = List.of(new CodingToLiteral()); - final List sqlFunction2 = List.of(new DateTimeAddDurationFunction(), - new DateTimeSubtractDurationFunction(), new DateAddDurationFunction(), - new DateSubtractDurationFunction(), - new DateTimeEqualsFunction(), new DateTimeGreaterThanFunction(), - new DateTimeGreaterThanOrEqualToFunction(), new DateTimeLessThanFunction(), - new DateTimeLessThanOrEqualToFunction(), new TimeEqualsFunction(), - new TimeGreaterThanFunction(), new TimeGreaterThanOrEqualToFunction(), - new TimeLessThanFunction(), new TimeLessThanOrEqualToFunction()); - final List sqlFunction3 = List.of(new TemporalDifferenceFunction()); return Spark.build(configuration, environment, sparkListener, sqlFunction1, sqlFunction2, sqlFunction3); } From e859884288a9843c18b083540d3290ae8cb27a19 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Fri, 9 Sep 2022 12:59:58 +0200 Subject: [PATCH 55/83] Refactoring: extracting out calendar duration related functions. --- .../au/csiro/pathling/fhirpath/UcumUtils.java | 76 +++++++++++++++++++ .../fhirpath/element/QuantityPath.java | 24 +++--- .../fhirpath/encoding/QuantityEncoding.java | 37 ++++++++- .../fhirpath/literal/QuantityLiteralPath.java | 44 +++++------ .../operator/DateArithmeticOperator.java | 6 +- .../sql/dates/TemporalArithmeticFunction.java | 13 +--- 6 files changed, 146 insertions(+), 54 deletions(-) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java new file mode 100644 index 0000000000..2ddd12a29e --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java @@ -0,0 +1,76 @@ +package au.csiro.pathling.fhirpath; + +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import com.google.common.collect.ImmutableMap; +import org.hl7.fhir.r4.model.Quantity; +import javax.annotation.Nonnull; +import java.math.BigDecimal; +import java.util.Calendar; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static au.csiro.pathling.utilities.Preconditions.checkUserInput; + +public final class UcumUtils { + + private UcumUtils() { + // Toolkit class + } + + + public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; + private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); + + private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder().put( + "year", Calendar.YEAR).put("years", Calendar.YEAR).put("month", Calendar.MONTH) + .put("months", Calendar.MONTH).put("day", Calendar.DATE).put("days", Calendar.DATE) + .put("hour", Calendar.HOUR).put("hours", Calendar.HOUR).put("minute", Calendar.MINUTE) + .put("minutes", Calendar.MINUTE).put("second", Calendar.SECOND) + .put("seconds", Calendar.SECOND).put("millisecond", Calendar.MILLISECOND) + .put("milliseconds", Calendar.MILLISECOND).build(); + + + public static int ucumToCalendarTemporalUnit(@Nonnull final String ucumTemporalUnit) { + final Integer temporalUnit = CALENDAR_DURATION_TO_UCUM.get( + ucumTemporalUnit); + checkUserInput(temporalUnit != null, + "Unsupported calendar duration unit: " + ucumTemporalUnit); + return temporalUnit; + } + + public static boolean isCalendarDuration(@Nonnull final Quantity maybeCalendarDuration) { + return maybeCalendarDuration.getSystem() + .equals(FHIRPATH_CALENDAR_DURATION_URI); + } + + @Nonnull + public static Quantity ensureCalendarDuration(@Nonnull final Quantity maybeCalendarDuration) { + if (!isCalendarDuration(maybeCalendarDuration)) { + throw new IllegalArgumentException("Calendar duration must have a system of " + + FHIRPATH_CALENDAR_DURATION_URI); + } + return maybeCalendarDuration; + } + + public static int getTemporalUnit(@Nonnull final Quantity calendarDuration) { + return ucumToCalendarTemporalUnit(ensureCalendarDuration(calendarDuration).getCode()); + } + + @Nonnull + public static Quantity parseCalendarDuration(@Nonnull final String calendarDurationString) { + final Matcher matcher = CALENDAR_DURATION_PATTERN.matcher(calendarDurationString); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Calendar duration literal has invalid format: " + calendarDurationString); + } + final String value = matcher.group(1); + final String keyword = matcher.group(2); + + final Quantity calendarDuration = new Quantity(); + calendarDuration.setValue(new BigDecimal(value)); + calendarDuration.setSystem(FHIRPATH_CALENDAR_DURATION_URI); + calendarDuration.setCode(keyword); + return calendarDuration; + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 2ae130de92..7e7edd3bee 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -7,7 +7,6 @@ package au.csiro.pathling.fhirpath.element; import static org.apache.spark.sql.functions.lit; -import static org.apache.spark.sql.functions.struct; import static org.apache.spark.sql.functions.when; import au.csiro.pathling.fhirpath.Comparable; @@ -90,17 +89,18 @@ public static Function buildMathOperation(@Nonnull fina QuantityEncoding.CANONICALIZED_CODE_COLUMN); final Column targetCanonicalizedCode = targetContext.getField( QuantityEncoding.CANONICALIZED_CODE_COLUMN); - final Column resultStruct = struct( - sourceContext.getField("id").as("id"), - resultColumn.as("value"), - lit(null).as("value_scale"), - sourceContext.getField("comparator").as("comparator"), - sourceCanonicalizedCode.as("unit"), - sourceContext.getField("system").as("system"), - sourceCanonicalizedCode.as("code"), - resultColumn.as(QuantityEncoding.CANONICALIZED_VALUE_COLUMN), - sourceCanonicalizedCode.as(QuantityEncoding.CANONICALIZED_CODE_COLUMN), - sourceContext.getField("_fid").as("_fid") + final Column resultStruct = QuantityEncoding.toStruct( + sourceContext.getField("id"), + resultColumn, + // TODO: Compute scale correctly depending on the operation + lit(null), + sourceContext.getField("comparator"), + sourceCanonicalizedCode, + sourceContext.getField("system"), + sourceCanonicalizedCode, + resultColumn, + sourceCanonicalizedCode, + sourceContext.getField("_fid") ); final Column resultQuantityColumn = when(sourceCanonicalizedCode.equalTo(targetCanonicalizedCode), resultStruct) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 1510ca9688..ea07935224 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -6,14 +6,20 @@ package au.csiro.pathling.fhirpath.encoding; +import static org.apache.spark.sql.functions.lit; +import static org.apache.spark.sql.functions.struct; + +import au.csiro.pathling.encoders.QuantitySupport; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.functions; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.MetadataBuilder; @@ -24,8 +30,8 @@ public class QuantityEncoding { - public static final String CANONICALIZED_VALUE_COLUMN = "_value_canonicalized"; - public static final String CANONICALIZED_CODE_COLUMN = "_code_canonicalized"; + public static final String CANONICALIZED_VALUE_COLUMN = QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME(); + public static final String CANONICALIZED_CODE_COLUMN = QuantitySupport.CODE_CANONICALIZED_FIELD_NAME(); @Nullable public static Row encode(@Nullable final Quantity quantity) { @@ -94,5 +100,32 @@ public static StructType dataType() { new StructField[]{id, value, valueScale, comparator, unit, system, code, canonicalizedValue, canonicalizedCode, fid}); } + + @Nonnull + public static Column toStruct( + @Nonnull final Column id, + @Nonnull final Column value, + @Nonnull final Column value_scale, + @Nonnull final Column comparator, + @Nonnull final Column unit, + @Nonnull final Column system, + @Nonnull final Column code, + @Nonnull final Column canonicalizedValue, + @Nonnull final Column canonicalizedCode, + @Nonnull final Column _fid + ) { + return struct( + id.as("id"), + value.as("value"), + value_scale.as("value_scale"), + comparator.as("comparator"), + unit.as("unit"), + system.as("system"), + code.as("code"), + canonicalizedValue.as(CANONICALIZED_VALUE_COLUMN), + canonicalizedCode.as(CANONICALIZED_CODE_COLUMN), + _fid.as("_fid") + ); + } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index eea1fb268e..203947e491 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -15,6 +15,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; +import au.csiro.pathling.fhirpath.UcumUtils; import au.csiro.pathling.fhirpath.comparison.QuantitySqlComparator; import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; @@ -44,10 +45,9 @@ @Getter public class QuantityLiteralPath extends LiteralPath implements Comparable, Numeric { - public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; + //public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); - private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() .put("second", "s") @@ -105,20 +105,9 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, @Nonnull public static QuantityLiteralPath fromCalendarDurationString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) { - final Matcher matcher = CALENDAR_DURATION_PATTERN.matcher(fhirPath); - if (!matcher.matches()) { - throw new IllegalArgumentException( - "Calendar duration literal has invalid format: " + fhirPath); - } - final String value = matcher.group(1); - final String keyword = matcher.group(2); - final Quantity quantity = new Quantity(); - quantity.setValue(new BigDecimal(value)); - quantity.setSystem(FHIRPATH_CALENDAR_DURATION_URI); - quantity.setCode(keyword); - - return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), quantity, fhirPath); + return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), + UcumUtils.parseCalendarDuration(fhirPath), fhirPath); } private static BigDecimal getQuantityValue(final String value, final @Nonnull FhirPath context) { @@ -164,10 +153,11 @@ public Column buildValueColumn() { // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); - } else if (quantity.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI) && + } else if (UcumUtils.isCalendarDuration(quantity) && CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the // UCUM library to canonicalize the value and code. + // TODO: This needs to happen in Encoders too !!! final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); @@ -178,17 +168,17 @@ public Column buildValueColumn() { canonicalizedCode = null; } - return struct( - lit(quantity.getId()).as("id"), - lit(value).as("value"), - lit(value.scale()).as("value_scale"), - lit(comparator.map(QuantityComparator::toCode).orElse(null)).as("comparator"), - lit(quantity.getUnit()).as("unit"), - lit(quantity.getSystem()).as("system"), - lit(quantity.getCode()).as("code"), - lit(canonicalizedValue).as(QuantityEncoding.CANONICALIZED_VALUE_COLUMN), - lit(canonicalizedCode).as(QuantityEncoding.CANONICALIZED_CODE_COLUMN), - lit(null).as("_fid")); + return QuantityEncoding.toStruct( + lit(quantity.getId()), + lit(value), + lit(value.scale()), + lit(comparator.map(QuantityComparator::toCode).orElse(null)), + lit(quantity.getUnit()), + lit(quantity.getSystem()), + lit(quantity.getCode()), + lit(canonicalizedValue), + lit(canonicalizedCode), + lit(null)); } @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java index 7302983b3a..f021d01dac 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java @@ -14,6 +14,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; +import au.csiro.pathling.fhirpath.UcumUtils; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import javax.annotation.Nonnull; import org.apache.spark.sql.Dataset; @@ -45,11 +46,12 @@ public FhirPath invoke(@Nonnull final OperatorInput input) { final FhirPath right = input.getRight(); checkUserInput(left instanceof Temporal, type + " operator does not support left operand: " + left.getExpression()); + + // TODO: It does not seem to be strictly necessary for the right argument to be a literal. checkUserInput(right instanceof QuantityLiteralPath, type + " operator does not support right operand: " + right.getExpression()); final QuantityLiteralPath calendarDuration = (QuantityLiteralPath) right; - checkUserInput(calendarDuration.getValue().getSystem() - .equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI), + checkUserInput(UcumUtils.isCalendarDuration(calendarDuration.getValue()), "Right operand of " + type + " operator must be a calendar duration"); checkUserInput(left.isSingular(), "Left operand to " + type + " operator must be singular: " + left.getExpression()); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index dcf5f66b48..ad7e3f9a5e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -6,10 +6,8 @@ package au.csiro.pathling.sql.dates; -import static au.csiro.pathling.utilities.Preconditions.checkUserInput; - +import au.csiro.pathling.fhirpath.UcumUtils; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import au.csiro.pathling.sql.udf.SqlFunction2; import com.google.common.collect.ImmutableMap; import java.math.RoundingMode; @@ -52,16 +50,9 @@ protected T performSubtraction(@Nonnull final T temporal, @Nonnull private T performArithmetic(final @Nonnull T temporal, final @Nonnull Quantity calendarDuration, final boolean subtract) { - if (!calendarDuration.getSystem().equals(QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI)) { - throw new IllegalArgumentException("Calendar duration must have a system of " - + QuantityLiteralPath.FHIRPATH_CALENDAR_DURATION_URI); - } final int amountToAdd = calendarDuration.getValue().setScale(0, RoundingMode.HALF_UP) .intValue(); - final Integer temporalUnit = TemporalArithmeticFunction.CALENDAR_DURATION_TO_UCUM.get( - calendarDuration.getCode()); - checkUserInput(temporalUnit != null, - "Unsupported calendar duration unit: " + calendarDuration.getCode()); + final int temporalUnit = UcumUtils.getTemporalUnit(calendarDuration); @SuppressWarnings("unchecked") final T result = (T) temporal.copy(); result.add(temporalUnit, subtract From 893073d92df6bc75dbc0b4e4107823f0470994f9 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Mon, 12 Sep 2022 21:25:23 +0200 Subject: [PATCH 56/83] Refactoring: moved Quantity literal creation to QuantityEncoder. --- ...mUtils.java => CalendarDurationUtils.java} | 8 +- .../fhirpath/encoding/QuantityEncoding.java | 100 +++++++++++++++++- .../fhirpath/literal/QuantityLiteralPath.java | 60 ++--------- .../operator/DateArithmeticOperator.java | 4 +- .../sql/dates/TemporalArithmeticFunction.java | 4 +- 5 files changed, 115 insertions(+), 61 deletions(-) rename fhir-server/src/main/java/au/csiro/pathling/fhirpath/{UcumUtils.java => CalendarDurationUtils.java} (96%) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java similarity index 96% rename from fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java rename to fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java index 2ddd12a29e..13cb1864e4 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/UcumUtils.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java @@ -1,6 +1,5 @@ package au.csiro.pathling.fhirpath; -import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import com.google.common.collect.ImmutableMap; import org.hl7.fhir.r4.model.Quantity; import javax.annotation.Nonnull; @@ -12,13 +11,12 @@ import static au.csiro.pathling.utilities.Preconditions.checkUserInput; -public final class UcumUtils { +public final class CalendarDurationUtils { - private UcumUtils() { + private CalendarDurationUtils() { // Toolkit class } - - + public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index ea07935224..9ee9a81e0f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -13,26 +13,51 @@ import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; +import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; -import org.apache.spark.sql.functions; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.MetadataBuilder; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; +import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; -public class QuantityEncoding { +/** + * Object decoders/encoders for {@link Quantity}. + */ +public final class QuantityEncoding { + + private QuantityEncoding() { + // Utility class + } + + private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() + .put("second", "s") + .put("seconds", "s") + .put("millisecond", "ms") + .put("milliseconds", "ms") + .build(); public static final String CANONICALIZED_VALUE_COLUMN = QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME(); public static final String CANONICALIZED_CODE_COLUMN = QuantitySupport.CODE_CANONICALIZED_FIELD_NAME(); + + /** + * Encodes a Quantity to a Row (spark SQL compatible type) + * + * @param quantity a coding to encode + * @return the Row representation of the quantity + */ @Nullable public static Row encode(@Nullable final Quantity quantity) { if (quantity == null) { @@ -48,6 +73,12 @@ public static Row encode(@Nullable final Quantity quantity) { comparator, quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */); } + /** + * Decodes a Quantity from a Row. + * + * @param row the row to decode + * @return the resulting Quantity + */ @Nonnull public static Quantity decode(@Nonnull final Row row) { final Quantity quantity = new Quantity(); @@ -76,6 +107,10 @@ public static Quantity decode(@Nonnull final Row row) { return quantity; } + + /** + * A {@link StructType} for a Quantity. + */ @Nonnull public static StructType dataType() { final Metadata metadata = new MetadataBuilder().build(); @@ -100,7 +135,22 @@ public static StructType dataType() { new StructField[]{id, value, valueScale, comparator, unit, system, code, canonicalizedValue, canonicalizedCode, fid}); } - + + /** + * Creates the structure representing the quantity column from its fields. + * + * @param id the id column + * @param value the value column + * @param value_scale the scale of the value column + * @param comparator the comparator column + * @param unit the unit column + * @param system the system column + * @param code the code column + * @param canonicalizedValue the canonicalized value column + * @param canonicalizedCode the canonicalized code column + * @param _fid the _fid column + * @return the SQL struct for the Quantity type. + */ @Nonnull public static Column toStruct( @Nonnull final Column id, @@ -128,4 +178,48 @@ public static Column toStruct( ); } + /** + * Encodes the quantity as a literal column that includes appropriate canonicalization. + * + * @param quantity the quantity to encode. + * @return the column with the literal representation of the quantity. + */ + @Nonnull + public static Column encodeLiteral(@Nonnull final Quantity quantity) { + final Optional comparator = Optional.ofNullable(quantity.getComparator()); + final BigDecimal value = quantity.getValue(); + + final BigDecimal canonicalizedValue; + final String canonicalizedCode; + if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { + // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. + canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); + canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); + } else if (CalendarDurationUtils.isCalendarDuration(quantity) && + CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { + // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the + // UCUM library to canonicalize the value and code. + final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); + canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); + canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); + } else { + // If it is neither a UCUM Quantity nor a calendar duration, it will not have a canonicalized + // form available. + canonicalizedValue = null; + canonicalizedCode = null; + } + + return toStruct( + lit(quantity.getId()), + lit(value), + lit(value.scale()), + lit(comparator.map(QuantityComparator::toCode).orElse(null)), + lit(quantity.getUnit()), + lit(quantity.getSystem()), + lit(quantity.getCode()), + lit(canonicalizedValue), + lit(canonicalizedCode), + lit(null)); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 203947e491..420a4575be 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -6,7 +6,6 @@ package au.csiro.pathling.fhirpath.literal; -import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.struct; import au.csiro.pathling.encoders.terminology.ucum.Ucum; @@ -15,13 +14,11 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; -import au.csiro.pathling.fhirpath.UcumUtils; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.comparison.QuantitySqlComparator; import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; -import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.regex.Matcher; @@ -35,7 +32,6 @@ import org.fhir.ucum.UcumService; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; import org.hl7.fhir.r4.model.Quantity; -import org.hl7.fhir.r4.model.Quantity.QuantityComparator; /** * Represents a FHIRPath Quantity literal. @@ -49,13 +45,6 @@ public class QuantityLiteralPath extends LiteralPath implements Compar private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); - private static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder() - .put("second", "s") - .put("seconds", "s") - .put("millisecond", "ms") - .put("milliseconds", "ms") - .build(); - protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, @Nonnull final Quantity literalValue) { super(dataset, idColumn, literalValue); @@ -102,12 +91,20 @@ public static QuantityLiteralPath fromUcumString(@Nonnull final String fhirPath, return buildLiteralPath(decimalValue, unit, Optional.ofNullable(display), context, fhirPath); } + /** + * Returns a new instance, parsed from a FHIRPath literal representing a calendar duration. + * + * @param fhirPath the FHIRPath representation of the literal + * @param context an input context that can be used to build a {@link Dataset} to represent the + * literal A new instance of {@link QuantityLiteralPath} + * @see Time-valued quantities + */ @Nonnull public static QuantityLiteralPath fromCalendarDurationString(@Nonnull final String fhirPath, @Nonnull final FhirPath context) { return new QuantityLiteralPath(context.getDataset(), context.getIdColumn(), - UcumUtils.parseCalendarDuration(fhirPath), fhirPath); + CalendarDurationUtils.parseCalendarDuration(fhirPath), fhirPath); } private static BigDecimal getQuantityValue(final String value, final @Nonnull FhirPath context) { @@ -143,42 +140,7 @@ public String getExpression() { @Nonnull @Override public Column buildValueColumn() { - final Quantity quantity = getValue(); - final Optional comparator = Optional.ofNullable(quantity.getComparator()); - final BigDecimal value = quantity.getValue(); - - final BigDecimal canonicalizedValue; - final String canonicalizedCode; - if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { - // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. - canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); - canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); - } else if (UcumUtils.isCalendarDuration(quantity) && - CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { - // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the - // UCUM library to canonicalize the value and code. - // TODO: This needs to happen in Encoders too !!! - final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); - canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); - canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); - } else { - // If it is neither a UCUM Quantity nor a calendar duration, it will not have a canonicalized - // form available. - canonicalizedValue = null; - canonicalizedCode = null; - } - - return QuantityEncoding.toStruct( - lit(quantity.getId()), - lit(value), - lit(value.scale()), - lit(comparator.map(QuantityComparator::toCode).orElse(null)), - lit(quantity.getUnit()), - lit(quantity.getSystem()), - lit(quantity.getCode()), - lit(canonicalizedValue), - lit(canonicalizedCode), - lit(null)); + return QuantityEncoding.encodeLiteral(getValue()); } @Nonnull diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java index f021d01dac..9432f1ea7e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java @@ -14,7 +14,7 @@ import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; -import au.csiro.pathling.fhirpath.UcumUtils; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import javax.annotation.Nonnull; import org.apache.spark.sql.Dataset; @@ -51,7 +51,7 @@ public FhirPath invoke(@Nonnull final OperatorInput input) { checkUserInput(right instanceof QuantityLiteralPath, type + " operator does not support right operand: " + right.getExpression()); final QuantityLiteralPath calendarDuration = (QuantityLiteralPath) right; - checkUserInput(UcumUtils.isCalendarDuration(calendarDuration.getValue()), + checkUserInput(CalendarDurationUtils.isCalendarDuration(calendarDuration.getValue()), "Right operand of " + type + " operator must be a calendar duration"); checkUserInput(left.isSingular(), "Left operand to " + type + " operator must be singular: " + left.getExpression()); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index ad7e3f9a5e..2bb4c1bc6b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -6,7 +6,7 @@ package au.csiro.pathling.sql.dates; -import au.csiro.pathling.fhirpath.UcumUtils; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.sql.udf.SqlFunction2; import com.google.common.collect.ImmutableMap; @@ -52,7 +52,7 @@ private T performArithmetic(final @Nonnull T temporal, final @Nonnull Quantity c final boolean subtract) { final int amountToAdd = calendarDuration.getValue().setScale(0, RoundingMode.HALF_UP) .intValue(); - final int temporalUnit = UcumUtils.getTemporalUnit(calendarDuration); + final int temporalUnit = CalendarDurationUtils.getTemporalUnit(calendarDuration); @SuppressWarnings("unchecked") final T result = (T) temporal.copy(); result.add(temporalUnit, subtract From d2e9d18e946ba257d9e9be11e40ee51fc4f619b7 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 13 Sep 2022 09:36:01 +0200 Subject: [PATCH 57/83] Updating java docs --- .../au/csiro/pathling/fhirpath/Temporal.java | 34 +++++++++++++++++++ .../fhirpath/function/UntilFunction.java | 7 ++++ .../csiro/pathling/sql/udf/SqlFunction.java | 14 ++++++++ .../csiro/pathling/sql/udf/SqlFunction1.java | 6 ++++ .../csiro/pathling/sql/udf/SqlFunction2.java | 7 ++++ .../csiro/pathling/sql/udf/SqlFunction3.java | 8 +++++ 6 files changed, 76 insertions(+) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java index d787a7362b..7461038d0e 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Temporal.java @@ -18,13 +18,47 @@ import org.apache.spark.sql.functions; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +/** + * Describes a path that represents a temporal value such as DateTime or Date, and can be the + * subject of date arithmetic operations involving time durations. + * + * @author John Grimes + */ public interface Temporal { + /** + * Gets a function that can take the {@link QuantityLiteralPath} representing a time duration and + * return a {@link FhirPath} that contains the result of date arithmetic operation for this path + * and the provided duration. The type of operation is controlled by supplying a {@link + * MathOperation}. + * + * @param operation The {@link MathOperation} type to retrieve a result for + * @param dataset The {@link Dataset} to use within the result + * @param expression The FHIRPath expression to use within the result + * @return A {@link Function} that takes a {@link QuantityLiteralPath} as its parameter, and + * returns a {@link FhirPath}. + */ @Nonnull Function getDateArithmeticOperation( @Nonnull MathOperation operation, @Nonnull Dataset dataset, @Nonnull String expression); + /** + * Gets a function that can take the {@link QuantityLiteralPath} representing a time duration and + * return a {@link FhirPath} that contains the result of applying the date arithmetic operation + * for to the source path and the provided duration. The type of operation is controlled by + * supplying a {@link MathOperation}. + * + * @param source the {@link FhirPath} to which the operation should be applied to. Should be a + * {@link Temporal} path. + * @param operation The {@link MathOperation} type to retrieve a result for + * @param dataset The {@link Dataset} to use within the result + * @param expression the FHIRPath expression to use within the result + * @param additionFunctionName the name of the UDF to use for additions. + * @param subtractionFunctionName the name of the UDF to use for subtractions. + * @return A {@link Function} that takes a {@link QuantityLiteralPath} as its parameter, and + * returns a {@link FhirPath}. + */ @Nonnull static Function buildDateArithmeticOperation( @Nonnull final FhirPath source, final @Nonnull MathOperation operation, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java index 4ae551694c..3adc613518 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java @@ -26,6 +26,13 @@ import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +/** + * This function computes the time interval (duration) between two paths representing dates or dates + * with time. + * + * @author John Grimes + * @see until + */ public class UntilFunction implements NamedFunction { private static final String NAME = "until"; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java index 5f4735f94a..f593078c56 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction.java @@ -8,10 +8,24 @@ import org.apache.spark.sql.types.DataType; +/** + * The interface that encapsulates meta data required for registration of a custom UDF function in + * Spark. + */ public interface SqlFunction { + /** + * Gets the name of the UDF. + * + * @return the name of the UDF function. + */ String getName(); + /** + * Gets the return type of the UDF + * + * @return the SQL type returned by the UDF. + */ DataType getReturnType(); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java index 8ecf9f8755..990ca34892 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction1.java @@ -8,6 +8,12 @@ import org.apache.spark.sql.api.java.UDF1; +/** + * A registrable UDF function with one argument. + * + * @param the type of the argument. + * @param the type of the result. + */ public interface SqlFunction1 extends SqlFunction, UDF1 { } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java index cdff936922..055ad09cd6 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction2.java @@ -8,6 +8,13 @@ import org.apache.spark.sql.api.java.UDF2; +/** + * A registrable UDF function with two arguments. + * + * @param the type of the first argument. + * @param the type of the second argument. + * @param the type of the result. + */ public interface SqlFunction2 extends SqlFunction, UDF2 { } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java index 034429a9ce..c41d03128c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/udf/SqlFunction3.java @@ -8,6 +8,14 @@ import org.apache.spark.sql.api.java.UDF3; +/** + * A registrable UDF function with three arguments. + * + * @param the type of the first argument. + * @param the type of the second argument. + * @param the type of the third argument. + * @param the type of the result. + */ public interface SqlFunction3 extends SqlFunction, UDF3 { } From af4ec1015c8431d9dd872a40af0e8289eb4a799c Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 13 Sep 2022 19:26:03 +0200 Subject: [PATCH 58/83] Make 'until' function require singular object and argument (in the same way as for arithmetic operators). --- .../fhirpath/function/UntilFunction.java | 16 +++++++- .../fhirpath/function/UntilFunctionTest.java | 38 ++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java index 3adc613518..fa56b0832f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/function/UntilFunction.java @@ -7,6 +7,8 @@ package au.csiro.pathling.fhirpath.function; import static au.csiro.pathling.QueryHelpers.join; +import static au.csiro.pathling.fhirpath.NonLiteralPath.findEidColumn; +import static au.csiro.pathling.fhirpath.NonLiteralPath.findThisColumn; import static au.csiro.pathling.utilities.Preconditions.checkUserInput; import static org.apache.spark.sql.functions.callUDF; @@ -25,6 +27,7 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import java.util.Optional; /** * This function computes the time interval (duration) between two paths representing dates or dates @@ -51,6 +54,12 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { checkUserInput(toArgument instanceof DateTimePath || toArgument instanceof DateTimeLiteralPath || toArgument instanceof DatePath || toArgument instanceof DateLiteralPath, "until function must have a DateTime or Date as the first argument"); + + checkUserInput(fromArgument.isSingular(), + "until function must be invoked on a singular path"); + checkUserInput(toArgument.isSingular(), + "until function must have the singular path as its first argument"); + checkUserInput(calendarDurationArgument instanceof StringLiteralPath, "until function must have a String as the second argument"); final String literalValue = ((StringLiteralPath) calendarDurationArgument).getValue() @@ -65,9 +74,12 @@ public FhirPath invoke(@Nonnull final NamedFunctionInput input) { calendarDurationArgument.getValueColumn()); final String expression = NamedFunction.expressionFromInput(input, NAME); + final Optional eidColumn = findEidColumn(fromArgument, toArgument); + final Optional thisColumn = findThisColumn(fromArgument, toArgument); + return ElementPath.build(expression, dataset, fromArgument.getIdColumn(), - fromArgument.getEidColumn(), valueColumn, fromArgument.isSingular(), - fromArgument.getCurrentResource(), fromArgument.getThisColumn(), FHIRDefinedType.INTEGER); + eidColumn, valueColumn, true, + fromArgument.getCurrentResource(), thisColumn, FHIRDefinedType.INTEGER); } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java index 2b0dc2d77b..1e78f5efd1 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/function/UntilFunctionTest.java @@ -11,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.FhirPath; @@ -152,11 +153,13 @@ Stream parameters() { .fhirType(FHIRDefinedType.DATETIME) .dataset(leftDataset()) .idAndValueColumns() + .singular(true) .build(); final ElementPath argument = new ElementPathBuilder(spark) .fhirType(FHIRDefinedType.DATETIME) .dataset(rightDataset()) .idAndValueColumns() + .singular(true) .build(); final ParserContext context = new ParserContextBuilder(spark, fhirContext) .groupingColumns(Collections.singletonList(input.getIdColumn())) @@ -206,11 +209,13 @@ void milliseconds() { .fhirType(FHIRDefinedType.DATE) .dataset(leftDataset) .idAndValueColumns() + .singular(true) .build(); final ElementPath argument = new ElementPathBuilder(spark) .fhirType(FHIRDefinedType.DATE) .dataset(rightDataset) .idAndValueColumns() + .singular(true) .build(); final ParserContext context = new ParserContextBuilder(spark, fhirContext) .groupingColumns(Collections.singletonList(input.getIdColumn())) @@ -245,6 +250,7 @@ void dateLiteralArgument() throws ParseException { .fhirType(FHIRDefinedType.DATE) .dataset(leftDataset) .idAndValueColumns() + .singular(true) .build(); final DateLiteralPath argument = DateLiteralPath.fromString("2020-01-02", input); final ParserContext context = new ParserContextBuilder(spark, fhirContext) @@ -276,6 +282,7 @@ void dateTimeLiteralArgument() throws ParseException { .fhirType(FHIRDefinedType.DATE) .dataset(leftDataset) .idAndValueColumns() + .singular(true) .build(); final DateTimeLiteralPath argument = DateTimeLiteralPath.fromString("2020-01-02T00:00:00Z", input); @@ -308,10 +315,15 @@ void invalidCalendarDuration() { .fhirType(FHIRDefinedType.DATE) .dataset(leftDataset) .idAndValueColumns() + .singular(true) .build(); + + final DateTimePath argument = mock(DateTimePath.class); + when(argument.isSingular()).thenReturn(true); + final NamedFunctionInput functionInput = new NamedFunctionInput(mock(ParserContext.class), input, - List.of(mock(DateTimePath.class), StringLiteralPath.fromString("nanosecond", input))); + List.of(argument, StringLiteralPath.fromString("nanosecond", input))); final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, () -> NamedFunction.getInstance("until").invoke(functionInput)); assertEquals("Invalid calendar duration: nanosecond", error.getMessage()); @@ -345,4 +357,28 @@ void wrongArgumentType() { error.getMessage()); } + @Test + void inputNotSingular() { + final DateTimePath input = mock(DateTimePath.class); + final NamedFunctionInput functionInput = new NamedFunctionInput(mock(ParserContext.class), + input, List.of(mock(DateTimeLiteralPath.class), + StringLiteralPath.fromString("nanosecond", input))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(functionInput)); + assertEquals("until function must be invoked on a singular path", error.getMessage()); + } + + @Test + void argumentNotSingular() { + final DateTimePath input = mock(DateTimePath.class); + when(input.isSingular()).thenReturn(true); + final NamedFunctionInput functionInput = new NamedFunctionInput(mock(ParserContext.class), + input, + List.of(mock(DateTimeLiteralPath.class), + StringLiteralPath.fromString("nanosecond", input))); + final InvalidUserInputError error = assertThrows(InvalidUserInputError.class, + () -> NamedFunction.getInstance("until").invoke(functionInput)); + assertEquals("until function must have the singular path as its first argument", + error.getMessage()); + } } From 587d8fda82836ceb0909f743139ab6939f6f68e4 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 13 Sep 2022 20:28:12 +0200 Subject: [PATCH 59/83] Cleanup (removing unnecessary fields, adding notes) --- .../au/csiro/pathling/fhirpath/element/QuantityPath.java | 3 ++- .../pathling/fhirpath/literal/QuantityLiteralPath.java | 2 -- .../pathling/sql/dates/TemporalArithmeticFunction.java | 8 -------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 7e7edd3bee..b0fd8265b4 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -92,7 +92,8 @@ public static Function buildMathOperation(@Nonnull fina final Column resultStruct = QuantityEncoding.toStruct( sourceContext.getField("id"), resultColumn, - // TODO: Compute scale correctly depending on the operation + // NOTE: This (setting value_scale to null) works because we never decode this struct to a Quantity. + // The only Quantities that are decoded are calendar duration quantities parsed from literals. lit(null), sourceContext.getField("comparator"), sourceCanonicalizedCode, diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index 420a4575be..c94f28e8d8 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -41,8 +41,6 @@ @Getter public class QuantityLiteralPath extends LiteralPath implements Comparable, Numeric { - //public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; - private static final Pattern UCUM_PATTERN = Pattern.compile("([0-9.]+) ('[^']+')"); protected QuantityLiteralPath(@Nonnull final Dataset dataset, @Nonnull final Column idColumn, diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index 2bb4c1bc6b..367be3dcbf 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -28,14 +28,6 @@ public abstract class TemporalArithmeticFunction imp private static final long serialVersionUID = -5016153440496309996L; - static final Map CALENDAR_DURATION_TO_UCUM = new ImmutableMap.Builder().put( - "year", Calendar.YEAR).put("years", Calendar.YEAR).put("month", Calendar.MONTH) - .put("months", Calendar.MONTH).put("day", Calendar.DATE).put("days", Calendar.DATE) - .put("hour", Calendar.HOUR).put("hours", Calendar.HOUR).put("minute", Calendar.MINUTE) - .put("minutes", Calendar.MINUTE).put("second", Calendar.SECOND) - .put("seconds", Calendar.SECOND).put("millisecond", Calendar.MILLISECOND) - .put("milliseconds", Calendar.MILLISECOND).build(); - @Nonnull protected T performAddition(@Nonnull final T temporal, @Nonnull final Quantity calendarDuration) { return performArithmetic(temporal, calendarDuration, false); From 989c1e5dd21bc8da254f80ec3f6d7ffd25002ecb Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 20 Sep 2022 13:05:32 +0200 Subject: [PATCH 60/83] Added comparison and equality precision test. --- .../QuantityOperatorsPrecisionTest.java | 203 ++++++++++++++++++ .../pathling/test/helpers/SparkHelpers.java | 16 +- 2 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java new file mode 100644 index 0000000000..73b907046c --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java @@ -0,0 +1,203 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.fhirpath.operator; + +import au.csiro.pathling.fhirpath.FhirPath; +import au.csiro.pathling.fhirpath.element.ElementPath; +import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.fhirpath.parser.ParserContext; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.builders.ElementPathBuilder; +import au.csiro.pathling.test.builders.ParserContextBuilder; +import au.csiro.pathling.test.helpers.TestHelpers; +import ca.uhn.fhir.context.FhirContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; +import org.codehaus.jettison.badgerfish.BadgerFishXMLInputFactory; +import org.fhir.ucum.Prefix; +import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.Enumerations; +import org.hl7.fhir.r4.model.Enumerations.FHIRDefinedType; +import org.hl7.fhir.r4.model.Quantity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import javax.annotation.Nonnull; +import java.math.BigDecimal; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static au.csiro.pathling.test.assertions.Assertions.assertThat; +import static au.csiro.pathling.test.helpers.SparkHelpers.quantityStructType; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowForUcumQuantity; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; + +@SpringBootTest +@Tag("UnitTest") +public class QuantityOperatorsPrecisionTest { + + @Autowired + SparkSession spark; + + @Autowired + FhirContext fhirContext; + + @Autowired + UcumService ucumService; + + static final String ID_ALIAS = "_abc123"; + + // Reasonable decimal with units assume above the value of 9999 we would use the next prefix up + // (if available) + + static final String REASONABLE_DECIMAL_01 = createSpanningDecimal(9, 3, 1, + 6).toString(); // 9000.00001 + static final String REASONABLE_DECIMAL_02 = createSpanningDecimal(9, 3, 2, + 6).toString(); // 9000.00002 + + // for Decimal(32,6) + static final String FULL_DECIMAL_01 = createSpanningDecimal(9, 26, 1, + 6).toString(); // 9e26 + 0.00001 + static final String FULL_DECIMAL_02 = createSpanningDecimal(9, 26, 2, + 6).toString(); // 9e26 + 0.00002 + + @BeforeEach + void setUp() { + } + + + @Nonnull + private static String unitToRowId(@Nonnull final String unit) { + return "unit-" + unit; + } + + @Nonnull + private ElementPath buildQuantityPathForUnits(@Nonnull final String value, List units) { + DatasetBuilder datasetBuilder = new DatasetBuilder(spark) + .withIdColumn(ID_ALIAS) + .withStructTypeColumns(quantityStructType()); + for (String unit : units) { + datasetBuilder = datasetBuilder.withRow(unitToRowId(unit), rowForUcumQuantity(value, unit)); + } + final Dataset dataset = datasetBuilder.buildWithStructValue(); + return new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .dataset(dataset) + .idAndValueColumns() + .build(); + } + + @Nonnull + private List getAllPrefixedUnits(@Nonnull final String baseUnit) { + return ucumService.getModel().getPrefixes().stream() + .map(Prefix::getCode) + .filter(p -> p.length() == 1) // filter out Ki, Gi etc + .map(p -> p + baseUnit) + .collect(Collectors.toUnmodifiableList()); + } + + @Nonnull + private static BigDecimal createSpanningDecimal(int leftValue, int leftScale, int rightValue, + int rightScale) { + return new BigDecimal(leftValue).movePointRight(leftScale) + .add(new BigDecimal(rightValue).movePointLeft(rightScale)); + } + + @Nonnull + private static List createResult(@Nonnull final List unitRange, boolean result) { + return unitRange.stream().map( + unit -> + RowFactory.create(unitToRowId(unit), result)).collect(Collectors.toList()); + } + + @Nonnull + private FhirPath callOperator(@Nonnull final ElementPath left, @Nonnull final String operator, + @Nonnull final ElementPath right) { + final ParserContext parserContext = new ParserContextBuilder(spark, fhirContext) + .groupingColumns(Collections.singletonList(left.getIdColumn())) + .build(); + + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator equalityOperator = Operator.getInstance(operator); + return equalityOperator.invoke(input); + } + + @Test + void equalityPrecisionForReasonableDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final FhirPath result = callOperator(left, "=", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + + @Test + void nonEqualityPrecisionForReasonableDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(REASONABLE_DECIMAL_02, unitRange); + final FhirPath result = callOperator(left, "!=", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + + @Test + void comparisonPrecisionForReasonableDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(REASONABLE_DECIMAL_02, unitRange); + final FhirPath result = callOperator(left, "<", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + + @Test + void equalityPrecisionForFullDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); + final FhirPath result = callOperator(left, "=", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + @Test + void nonEqualityPrecisionForFullDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); + left.getDataset().collectAsList().forEach(System.out::println); + final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_02, unitRange); + right.getDataset().collectAsList().forEach(System.out::println); + final FhirPath result = callOperator(left, "!=", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + @Test + void comparisonPrecisionForFullDecimals() { + final List unitRange = getAllPrefixedUnits("m"); + final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_02, unitRange); + final FhirPath result = callOperator(left, "<", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + + @Test + void equalityPrecisionForReasonableDecimalsWithMols() { + final List unitRange = getAllPrefixedUnits("mol"); + final ElementPath left = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final ElementPath right = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); + final FhirPath result = callOperator(left, "=", right); + assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 8b27068b6c..89594f5715 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -180,15 +180,25 @@ public static Row rowForUcumQuantity(@Nonnull final BigDecimal value, quantity.setValue( quantity.getValue().setScale(DecimalCustomCoder.scale(), RoundingMode.HALF_UP)); } - if (quantity.getValue().precision() > DecimalCustomCoder.precision()) { - throw new AssertionError("Attempt to encode a value with greater than supported precision"); - } + // TODO: This is not really how it should work + // As the BigDecimal::precision is not same as DecimalType precision(). + // Besides this results in the value being set to null + // if (quantity.getValue().precision() > DecimalCustomCoder.precision()) { + // throw new AssertionError("Attempt to encode a value with greater than supported precision"); + // } quantity.setUnit(unit); quantity.setSystem(TestHelpers.UCUM_URL); quantity.setCode(unit); return rowFromQuantity(quantity); } + + @Nonnull + public static Row rowForUcumQuantity(@Nonnull final String value, + @Nonnull final String unit) { + return rowForUcumQuantity(new BigDecimal(value), unit); + } + @Value public static class IdAndValueColumns { From 330fc2d641bf50da0c8652c94beadedd03e9c926 Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Tue, 20 Sep 2022 21:15:42 +0200 Subject: [PATCH 61/83] Benchmark for two flexible decimal implementations. --- .../csiro/pathling/sql/types/FlexDecimal.java | 89 +++++++++ .../csiro/pathling/sql/types/UcumDecimal.java | 77 ++++++++ .../test/benchmark/DecimalBenchmark.java | 177 ++++++++++++++++++ 3 files changed, 343 insertions(+) create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java create mode 100644 fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java create mode 100644 fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java b/fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java new file mode 100644 index 0000000000..6414e57d8a --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java @@ -0,0 +1,89 @@ +package au.csiro.pathling.sql.types; + +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.api.java.UDF2; +import org.apache.spark.sql.expressions.UserDefinedFunction; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import javax.annotation.Nonnull; +import java.math.BigDecimal; + +public class FlexDecimal { + + @Nonnull + private static StructType createFlexibleDecimalType() { + final Metadata metadata = new MetadataBuilder().build(); + final StructField value = new StructField("value", DataTypes.createDecimalType(32, 0), true, + metadata); + final StructField scale = new StructField("scale", DataTypes.IntegerType, true, metadata); + return new StructType(new StructField[]{value, scale}); + } + + @Nonnull + public static DataType DATA_TYPE = createFlexibleDecimalType(); + + @Nonnull + private static UserDefinedFunction toBooleanUdf( + UDF2 method) { + final UDF2 f = (left, right) -> + method.call(fromRow(left), fromRow(right)); + return functions.udf(f, DataTypes.BooleanType); + } + + @Nonnull + private static UDF2 wrapBigDecimal2( + UDF2 method) { + return (left, right) -> toRow( + method.call(fromRow(left), fromRow(right))); + } + + @Nonnull + private static UserDefinedFunction toBigDecimalUdf( + UDF2 method) { + return functions.udf(wrapBigDecimal2(method), DATA_TYPE); + } + + @Nonnull + public static BigDecimal fromRow(@Nonnull final Row row) { + final BigDecimal unscaledValue = row.getDecimal(0); + return unscaledValue.movePointLeft(row.getInt(1)); + } + + @Nonnull + public static Row toRow(@Nonnull final BigDecimal decimal) { + return RowFactory.create(decimal.movePointRight(decimal.scale()), decimal.scale()); + } + + private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf(BigDecimal::equals); + private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) < 0); + + private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(BigDecimal::add); + private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(BigDecimal::multiply); + + @Nonnull + public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { + return EQUALS_UDF.apply(left, right); + } + + @Nonnull + public static Column lt(@Nonnull final Column left, @Nonnull final Column right) { + return LT_UDF.apply(left, right); + } + + @Nonnull + public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { + return PLUS_UDF.apply(left, right); + } + + @Nonnull + public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { + return MULTIPLY_UDF.apply(left, right); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java b/fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java new file mode 100644 index 0000000000..cc4f0c4abb --- /dev/null +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java @@ -0,0 +1,77 @@ +package au.csiro.pathling.sql.types; + +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.api.java.UDF2; +import org.apache.spark.sql.expressions.UserDefinedFunction; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.*; +import org.fhir.ucum.Decimal; +import org.fhir.ucum.UcumException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.math.BigDecimal; + +public class UcumDecimal { + + @Nonnull + public static DataType DATA_TYPE = DataTypes.StringType; + + @Nonnull + private static UserDefinedFunction toBooleanUdf( + UDF2 method) { + final UDF2 f = (left, right) -> + method.call(fromValue(left), fromValue(right)); + return functions.udf(f, DataTypes.BooleanType); + } + + @Nonnull + private static UserDefinedFunction toBigDecimalUdf( + UDF2 method) { + + final UDF2 f = (left, right) -> + toValue(method.call(fromValue(left), fromValue(right))); + return functions.udf(f, DATA_TYPE); + } + + @Nullable + public static Decimal fromValue(@Nonnull final String value) { + try { + return new Decimal(value); + } catch (UcumException e) { + return null; + } + } + + @Nonnull + public static String toValue(@Nonnull final Decimal decimal) { + return decimal.toString(); + } + + private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf(Decimal::equals); + private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.comparesTo(r) < 0); + + private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(Decimal::add); + private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(Decimal::multiply); + + @Nonnull + public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { + return EQUALS_UDF.apply(left, right); + } + + @Nonnull + public static Column lt(@Nonnull final Column left, @Nonnull final Column right) { + return LT_UDF.apply(left, right); + } + + @Nonnull + public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { + return PLUS_UDF.apply(left, right); + } + + @Nonnull + public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { + return MULTIPLY_UDF.apply(left, right); + } +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java new file mode 100644 index 0000000000..f2a439410d --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -0,0 +1,177 @@ +package au.csiro.pathling.test.benchmark; + +import au.csiro.pathling.Configuration; +import au.csiro.pathling.aggregate.AggregateExecutor; +import au.csiro.pathling.aggregate.AggregateRequest; +import au.csiro.pathling.aggregate.AggregateRequestBuilder; +import au.csiro.pathling.aggregate.AggregateResponse; +import au.csiro.pathling.encoders.FhirEncoders; +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.fhir.TerminologyServiceFactory; +import au.csiro.pathling.io.Database; +import au.csiro.pathling.jmh.AbstractJmhSpringBootState; +import au.csiro.pathling.sql.types.FlexDecimal; +import au.csiro.pathling.sql.types.UcumDecimal; +import au.csiro.pathling.terminology.TerminologyService; +import au.csiro.pathling.test.SharedMocks; +import au.csiro.pathling.test.builders.DatasetBuilder; +import au.csiro.pathling.test.helpers.TestHelpers; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.parser.IParser; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.hl7.fhir.r4.model.Enumerations.ResourceType; +import org.junit.jupiter.api.Tag; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; +import javax.annotation.Nonnull; +import java.math.BigDecimal; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Tag("UnitTest") +@Fork(0) +@Warmup(iterations = 3) +@Measurement(iterations = 7) +public class DecimalBenchmark { + + private static final int ROWS = 1000000; + private static final BigDecimal LEFT_DECIMAL = new BigDecimal("1493493840938434300000.123456"); + private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("999999999999138409380.123456"); + + @State(Scope.Benchmark) + @ActiveProfiles("unit-test") + public static class DatasetState extends AbstractJmhSpringBootState { + + @Autowired + SparkSession spark; + + Dataset dataset; + + + @Setup(Level.Trial) + public void setUp() { + + DatasetBuilder datasetBuilder = new DatasetBuilder(spark) + .withColumn("leftDecimal", DecimalCustomCoder.decimalType()) + .withColumn("rightDecimal", DecimalCustomCoder.decimalType()) + .withColumn("leftFlexDecimal", FlexDecimal.DATA_TYPE) + .withColumn("rightFlexDecimal", FlexDecimal.DATA_TYPE) + .withColumn("leftUcumDecimal", UcumDecimal.DATA_TYPE) + .withColumn("rightUcumDecimal", UcumDecimal.DATA_TYPE); + + for (int i = 0; i < ROWS; i++) { + datasetBuilder = datasetBuilder.withRow( + LEFT_DECIMAL, + RIGHT_DECIMAL, + FlexDecimal.toRow(LEFT_DECIMAL), + FlexDecimal.toRow(RIGHT_DECIMAL), + LEFT_DECIMAL.toString(), + RIGHT_DECIMAL.toString() + ); + } + dataset = datasetBuilder.build().cache(); + } + + @Nonnull + Column col(String name) { + return dataset.col(name); + } + + @Nonnull + List collectQuery(Column col) { + return dataset.select(col).collectAsList(); + } + } + + @Benchmark + public void multiply_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").multiply(ds.col("rightDecimal")))); + } + + @Benchmark + public void add_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").plus(ds.col("rightDecimal")))); + } + + @Benchmark + public void equals_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").equalTo(ds.col("rightDecimal")))); + } + + @Benchmark + public void lt_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").lt(ds.col("rightDecimal")))); + } + + + @Benchmark + public void multiply_flexDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexDecimal.multiply(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); + } + + @Benchmark + public void add_flexDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexDecimal.plus(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); + } + + @Benchmark + public void equals_flexDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexDecimal.equals(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); + } + + @Benchmark + public void lt_flexDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexDecimal.lt(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); + } + + + @Benchmark + public void multiply_ucumDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + UcumDecimal.multiply(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); + } + + @Benchmark + public void add_ucumDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + UcumDecimal.plus(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); + } + + @Benchmark + public void equals_ucumDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + UcumDecimal.equals(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); + } + + @Benchmark + public void lt_ucumDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + UcumDecimal.lt(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); + } +} From ecd67beaefd4d58c93a763b19649d0bd5635f30f Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Fri, 7 Oct 2022 16:41:26 +0200 Subject: [PATCH 62/83] Switched canonicalized value to FlexDecimal based on string. --- .../encoders/terminology/ucum/Ucum.java | 51 +++++-- .../csiro/pathling/sql/types/FlexDecimal.java | 117 ++++++++++++++++ .../pathling/sql/types/FlexiDecimal.java | 19 +-- .../csiro/pathling/sql/types/UcumDecimal.java | 0 .../pathling/encoders/QuantitySupport.scala | 29 +++- .../encoders/LightweightFhirEncodersTest.java | 13 +- .../encoders/SchemaConverterTest.java | 3 +- .../au/csiro/pathling/fhirpath/FhirPath.java | 10 ++ .../comparison/QuantitySqlComparator.java | 27 ++-- .../fhirpath/element/QuantityPath.java | 109 ++++++++++++--- .../fhirpath/encoding/QuantityEncoding.java | 60 ++++++-- .../fhirpath/literal/QuantityLiteralPath.java | 2 +- .../EqualityOperatorQuantityTest.java | 130 +++++++++++++++--- .../operator/MathOperatorQuantityTest.java | 37 +++-- .../QuantityOperatorsPrecisionTest.java | 4 +- .../pathling/fhirpath/parser/ParserTest.java | 36 +++++ .../test/benchmark/DecimalBenchmark.java | 54 +++++--- .../pathling/test/helpers/SparkHelpers.java | 34 ++--- ...QuantityAdditionSubtractionAndEquality.csv | 9 ++ .../testQuantityAdditionWithOverflow_code.csv | 9 ++ ...testQuantityAdditionWithOverflow_value.csv | 9 ++ .../testQuantityMultiplicationAndDivision.csv | 9 ++ 22 files changed, 610 insertions(+), 161 deletions(-) create mode 100644 encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java rename fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java => encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java (81%) rename {fhir-server => encoders}/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java (100%) create mode 100644 fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionSubtractionAndEquality.csv create mode 100644 fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_code.csv create mode 100644 fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_value.csv create mode 100644 fhir-server/src/test/resources/responses/ParserTest/testQuantityMultiplicationAndDivision.csv diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java index 18962f3b3b..7f1fc0b375 100644 --- a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java +++ b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java @@ -31,6 +31,8 @@ */ public class Ucum { + public static final String NO_UNIT_CODE = "1"; + public static final String SYSTEM_URI = "http://unitsofmeasure.org"; private static final UcumService service; @@ -51,8 +53,8 @@ public static UcumService service() throws UcumException { } @Nullable - public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, - @Nonnull final String code) { + public static BigDecimal getCanonicalValue(@Nullable final BigDecimal value, + @Nullable final String code) { try { @Nullable final Pair result = getCanonicalForm(value, code); if (result == null) { @@ -73,8 +75,27 @@ public static BigDecimal getCanonicalValue(@Nonnull final BigDecimal value, } @Nullable - public static String getCanonicalCode(@Nonnull final BigDecimal value, - @Nonnull final String code) { + public static String getCanonicalValueAsString(@Nullable final BigDecimal value, + @Nullable final String code) { + try { + @Nullable final Pair result = getCanonicalForm(value, code); + if (result == null) { + return null; + } + @Nullable final Decimal decimalValue = result.getValue(); + if (decimalValue == null) { + return null; + } + @Nullable final String stringValue = decimalValue.asDecimal(); + return stringValue; + } catch (final UcumException e) { + return null; + } + } + + @Nullable + public static String getCanonicalCode(@Nullable final BigDecimal value, + @Nullable final String code) { try { @Nullable final Pair result = getCanonicalForm(value, code); if (result == null) { @@ -87,12 +108,24 @@ public static String getCanonicalCode(@Nonnull final BigDecimal value, } @Nullable - private static Pair getCanonicalForm(final @Nonnull BigDecimal value, final @Nonnull String code) + private static Pair getCanonicalForm(final @Nullable BigDecimal value, + final @Nullable String code) throws UcumException { - final int maxPrecision = DecimalCustomCoder.decimalType().precision(); - final Decimal decimalValue = new Decimal(value.toPlainString(), maxPrecision); - @Nullable final Pair result = service.getCanonicalForm(new Pair(decimalValue, code)); - return result; + if (value == null || code == null) { + return null; + } + final Decimal decimalValue = new Decimal(value.toPlainString()); + return adjustNoUnitCode(service.getCanonicalForm(new Pair(decimalValue, code))); + } + + @Nullable + private static Pair adjustNoUnitCode(@Nullable Pair pair) { + if (pair == null) { + return null; + } + return (pair.getCode() != null && pair.getCode().isEmpty()) + ? new Pair(pair.getValue(), NO_UNIT_CODE) + : pair; } } diff --git a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java new file mode 100644 index 0000000000..a791a004e5 --- /dev/null +++ b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java @@ -0,0 +1,117 @@ +package au.csiro.pathling.sql.types; + +import java.math.BigDecimal; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.api.java.UDF2; +import org.apache.spark.sql.expressions.UserDefinedFunction; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; + +public class FlexDecimal { + + public static int MAX_PRECISION = 48; + + @Nonnull + public static DataType DATA_TYPE = DataTypes.StringType; + + @Nonnull + private static UserDefinedFunction toBooleanUdf( + UDF2 method) { + final UDF2 f = (left, right) -> + (left == null || right == null) + ? null + : method.call(fromValue(left), fromValue(right)); + return functions.udf(f, DataTypes.BooleanType); + } + + @Nonnull + private static UDF2 wrapBigDecimal2( + UDF2 method) { + return (left, right) -> + (left == null || right == null) + ? null + : toValue(method.call(fromValue(left), fromValue(right))); + } + + @Nonnull + private static UserDefinedFunction toBigDecimalUdf( + UDF2 method) { + return functions.udf(wrapBigDecimal2(method), DATA_TYPE); + } + + @Nullable + public static BigDecimal fromValue(@Nullable final String value) { + return value != null + ? new BigDecimal(value) + : null; + } + + @Nullable + public static String toValue(@Nullable final BigDecimal decimal) { + return decimal != null && decimal.precision() <= MAX_PRECISION + ? decimal.toPlainString() + : null; + } + + private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf((l, r) -> l.compareTo(r) == 0); + private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) < 0); + private static final UserDefinedFunction LTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) <= 0); + private static final UserDefinedFunction GT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) > 0); + private static final UserDefinedFunction GTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) >= 0); + + private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(BigDecimal::add); + private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(BigDecimal::multiply); + private static final UserDefinedFunction MINUS_UDF = toBigDecimalUdf(BigDecimal::subtract); + private static final UserDefinedFunction DIVIDE_UDF = toBigDecimalUdf(BigDecimal::divide); + + + @Nonnull + public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { + return EQUALS_UDF.apply(left, right); + } + + @Nonnull + public static Column lt(@Nonnull final Column left, @Nonnull final Column right) { + return LT_UDF.apply(left, right); + } + + @Nonnull + public static Column lte(@Nonnull final Column left, @Nonnull final Column right) { + return LTE_UDF.apply(left, right); + } + + @Nonnull + public static Column gt(@Nonnull final Column left, @Nonnull final Column right) { + return GT_UDF.apply(left, right); + } + + @Nonnull + public static Column gte(@Nonnull final Column left, @Nonnull final Column right) { + return GTE_UDF.apply(left, right); + } + + + @Nonnull + public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { + return PLUS_UDF.apply(left, right); + } + + @Nonnull + public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { + return MULTIPLY_UDF.apply(left, right); + } + + + @Nonnull + public static Column minus(@Nonnull final Column left, @Nonnull final Column right) { + return MINUS_UDF.apply(left, right); + } + + @Nonnull + public static Column divide(@Nonnull final Column left, @Nonnull final Column right) { + return DIVIDE_UDF.apply(left, right); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java similarity index 81% rename from fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java rename to encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java index 6414e57d8a..e581ca3a55 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java +++ b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java @@ -6,16 +6,11 @@ import org.apache.spark.sql.api.java.UDF2; import org.apache.spark.sql.expressions.UserDefinedFunction; import org.apache.spark.sql.functions; -import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.Metadata; -import org.apache.spark.sql.types.MetadataBuilder; -import org.apache.spark.sql.types.StructField; -import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.types.*; import javax.annotation.Nonnull; import java.math.BigDecimal; -public class FlexDecimal { +public class FlexiDecimal { @Nonnull private static StructType createFlexibleDecimalType() { @@ -33,15 +28,15 @@ private static StructType createFlexibleDecimalType() { private static UserDefinedFunction toBooleanUdf( UDF2 method) { final UDF2 f = (left, right) -> - method.call(fromRow(left), fromRow(right)); + method.call(fromValue(left), fromValue(right)); return functions.udf(f, DataTypes.BooleanType); } @Nonnull private static UDF2 wrapBigDecimal2( UDF2 method) { - return (left, right) -> toRow( - method.call(fromRow(left), fromRow(right))); + return (left, right) -> toValue( + method.call(fromValue(left), fromValue(right))); } @Nonnull @@ -51,13 +46,13 @@ private static UserDefinedFunction toBigDecimalUdf( } @Nonnull - public static BigDecimal fromRow(@Nonnull final Row row) { + public static BigDecimal fromValue(@Nonnull final Row row) { final BigDecimal unscaledValue = row.getDecimal(0); return unscaledValue.movePointLeft(row.getInt(1)); } @Nonnull - public static Row toRow(@Nonnull final BigDecimal decimal) { + public static Row toValue(@Nonnull final BigDecimal decimal) { return RowFactory.create(decimal.movePointRight(decimal.scale()), decimal.scale()); } diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java similarity index 100% rename from fhir-server/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java rename to encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala index 99a1ac61d5..13b7c13b62 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala @@ -15,6 +15,7 @@ package au.csiro.pathling.encoders import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder import au.csiro.pathling.encoders.terminology.ucum.Ucum +import au.csiro.pathling.sql.types.FlexDecimal import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType, StructField} @@ -39,11 +40,27 @@ object QuantitySupport { def createExtraSerializers(expression: Expression): Seq[(String, Expression)] = { val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) - val canonicalizedValue = StaticInvoke(classOf[Decimal], - DecimalCustomCoder.decimalType, - "apply", - StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), - "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) + // TODO: Change to FlexDecimal + // val canonicalizedValue = StaticInvoke(classOf[Decimal], + // DecimalCustomCoder.decimalType, + // "apply", + // StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), + // "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) + + +// val struct = CreateNamedStruct( +// allFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) +// If(IsNull(expression), Literal.create(null, struct.dataType), struct) + + + val canonicalizedValue = + StaticInvoke( + classOf[UTF8String], + DataTypes.StringType, + "fromString", + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), + "getCanonicalValueAsString", Seq(valueExp, codeExp)) :: Nil) + val canonicalizedCode = StaticInvoke( classOf[UTF8String], @@ -64,7 +81,7 @@ object QuantitySupport { */ def createExtraSchemaFields(): Seq[StructField] = { Seq( - StructField(VALUE_CANONICALIZED_FIELD_NAME, DecimalCustomCoder.decimalType), + StructField(VALUE_CANONICALIZED_FIELD_NAME, FlexDecimal.DATA_TYPE), StructField(CODE_CANONICALIZED_FIELD_NAME, DataTypes.StringType) ) } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index 562e96553d..1d93fdb16b 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -100,12 +100,13 @@ public void assertSingletNestedExtension(final String expectedUrl, private static void assertQuantity(final Row quantityRow, final String canonicalizedValue, final String canonicalizedCode) { - final BigDecimal actualCanonicalizedValue = quantityRow.getDecimal( + // TODO: FlexDecimal + final String actualCanonicalizedValue = quantityRow.getString( quantityRow.fieldIndex(QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME())); final String actualCanonicalizedCode = quantityRow.getString( quantityRow.fieldIndex(QuantitySupport.CODE_CANONICALIZED_FIELD_NAME())); - assertEquals(new BigDecimal(canonicalizedValue), actualCanonicalizedValue); + assertEquals(canonicalizedValue, actualCanonicalizedValue); assertEquals(canonicalizedCode, actualCanonicalizedCode); } @@ -234,7 +235,7 @@ public void testQuantityCanonicalization() { final Row observationRow = rowEncoder.createDeserializer().apply(serializedRow); final Row quantityRow = observationRow.getStruct(observationRow.fieldIndex("valueQuantity")); - assertQuantity(quantityRow, "76000.000000", "g"); + assertQuantity(quantityRow, "76000", "g"); } @Test @@ -253,12 +254,12 @@ public void testQuantityArrayCanonicalization() { final List properties = deviceRow.getList(deviceRow.fieldIndex("property")); final Row propertyRow = properties.get(0); final List quantityArray = propertyRow.getList(propertyRow.fieldIndex("valueQuantity")); - + final Row quantity1 = quantityArray.get(0); - assertQuantity(quantity1, "0.001000", "m"); + assertQuantity(quantity1, "0.0010", "m"); final Row quantity2 = quantityArray.get(1); - assertQuantity(quantity2, "0.002000", "m"); + assertQuantity(quantity2, "0.0020", "m"); } } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java index ecf86e9b1d..fc530a32aa 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java @@ -417,7 +417,8 @@ private void assertQuantityType(final DataType quantityType) { assertTrue(getField(quantityType, true, "unit") instanceof StringType); assertTrue(getField(quantityType, true, "system") instanceof StringType); assertTrue(getField(quantityType, true, "code") instanceof StringType); - assertTrue(getField(quantityType, true, "_value_canonicalized") instanceof DecimalType); + // TODO: FlexDecimal Change + assertTrue(getField(quantityType, true, "_value_canonicalized") instanceof StringType); assertTrue(getField(quantityType, true, "_code_canonicalized") instanceof StringType); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java index ce5840304f..251663f711 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java @@ -95,4 +95,14 @@ NonLiteralPath combineWith(@Nonnull FhirPath target, @Nonnull Dataset datas @Nonnull String expression, @Nonnull Column idColumn, @Nonnull Optional eidColumn, @Nonnull Column valueColumn, boolean singular, @Nonnull Optional thisColumn); + + /** + * Prints out to stdout all the ids and values of all the elements in this path. For debugging + * purposes only. + */ + default void dumpAll() { + getDataset().select(getIdColumn(), getValueColumn()).collectAsList() + .forEach(System.out::println); + } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java index 2b894d6a4a..d68222e4ac 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java @@ -10,6 +10,7 @@ import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; +import au.csiro.pathling.sql.types.FlexDecimal; import org.apache.spark.sql.Column; import javax.annotation.Nonnull; import java.util.function.BiFunction; @@ -24,13 +25,9 @@ */ public class QuantitySqlComparator implements SqlComparator { - private final static QuantitySqlComparator INSTANCE = new QuantitySqlComparator( - STD_SQL_COMPARATOR); + private final static QuantitySqlComparator INSTANCE = new QuantitySqlComparator(); - private final SqlComparator defaultSqlComparator; - - public QuantitySqlComparator(final SqlComparator defaultSqlComparator) { - this.defaultSqlComparator = defaultSqlComparator; + public QuantitySqlComparator() { } private static BiFunction wrap( @@ -53,32 +50,32 @@ private static BiFunction wrap( @Override public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::equalsTo).apply(left, right); + return wrap(FlexDecimal::equals).apply(left, right); } - @Override - public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::notEqual).apply(left, right); - } + // @Override + // public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { + // return wrap(defaultSqlComparator::notEqual).apply(left, right); + // } @Override public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::lessThan).apply(left, right); + return wrap(FlexDecimal::lt).apply(left, right); } @Override public Column lessThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::lessThanOrEqual).apply(left, right); + return wrap(FlexDecimal::lte).apply(left, right); } @Override public Column greaterThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::greaterThan).apply(left, right); + return wrap(FlexDecimal::gt).apply(left, right); } @Override public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(defaultSqlComparator::greaterThanOrEqual).apply(left, right); + return wrap(FlexDecimal::gte).apply(left, right); } /** diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index b0fd8265b4..daad92e05c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -9,6 +9,7 @@ import static org.apache.spark.sql.functions.lit; import static org.apache.spark.sql.functions.when; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.NonLiteralPath; import au.csiro.pathling.fhirpath.Numeric; @@ -17,8 +18,10 @@ import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; +import au.csiro.pathling.sql.types.FlexDecimal; import com.google.common.collect.ImmutableSet; import java.util.Optional; +import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; import org.apache.spark.sql.Column; @@ -72,23 +75,91 @@ public Column getNumericContextColumn() { @Override public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { - return buildMathOperation(this, operation, expression, dataset, getFhirType()); + return buildMathOperation(this, operation, expression, dataset, getDefinition()); + } + + @Nonnull + private static BiFunction getMathOperation( + @Nonnull final MathOperation operation) { + switch (operation) { + case ADDITION: + return FlexDecimal::plus; + case MULTIPLICATION: + return FlexDecimal::multiply; + case DIVISION: + return FlexDecimal::divide; + case SUBTRACTION: + return FlexDecimal::minus; + default: + throw new AssertionError("Unsupported math operation encountered: " + operation); + } + } + + private static final Column NO_UNIT_LITERAL = lit(Ucum.NO_UNIT_CODE); + + @Nonnull + private static Column getResultUnit( + @Nonnull final MathOperation operation, @Nonnull final Column leftUnit, + @Nonnull final Column rightUnit) { + switch (operation) { + case ADDITION: + case SUBTRACTION: + return leftUnit; + case MULTIPLICATION: + // we only allow multiplication by dimensionless values at the moment + // the unit is preserved in this case + return when(leftUnit.notEqual(NO_UNIT_LITERAL), leftUnit).otherwise(rightUnit); + case DIVISION: + // we only allow division by the same unit or a dimensionless value + return when(leftUnit.equalTo(rightUnit), NO_UNIT_LITERAL).otherwise(leftUnit); + default: + throw new AssertionError("Unsupported math operation encountered: " + operation); + } + } + + @Nonnull + private static Column getValidResult( + @Nonnull final MathOperation operation, @Nonnull final Column result, + @Nonnull final Column leftUnit, + @Nonnull final Column rightUnit) { + switch (operation) { + case ADDITION: + case SUBTRACTION: + return when(leftUnit.equalTo(rightUnit), result) + .otherwise(null); + case MULTIPLICATION: + // we only allow multiplication by dimensionless values at the moment + // the unit is preserved in this case + return when(leftUnit.equalTo(NO_UNIT_LITERAL).or(rightUnit.equalTo(NO_UNIT_LITERAL)), + result).otherwise(null); + case DIVISION: + // we only allow division by the same unit or a dimensionless value + return when(leftUnit.equalTo(rightUnit).or(rightUnit.equalTo(NO_UNIT_LITERAL)), + result).otherwise(null); + default: + throw new AssertionError("Unsupported math operation encountered: " + operation); + } } @Nonnull public static Function buildMathOperation(@Nonnull final Numeric source, @Nonnull final MathOperation operation, @Nonnull final String expression, - @Nonnull final Dataset dataset, @Nonnull final FHIRDefinedType fhirType) { + @Nonnull final Dataset dataset, + @Nonnull final Optional elementDefinition) { return target -> { + final BiFunction mathOperation = getMathOperation(operation); final Column sourceComparable = source.getNumericValueColumn(); final Column sourceContext = source.getNumericContextColumn(); final Column targetContext = target.getNumericContextColumn(); - final Column resultColumn = operation.getSparkFunction() + final Column resultColumn = mathOperation .apply(sourceComparable, target.getNumericValueColumn()); final Column sourceCanonicalizedCode = sourceContext.getField( QuantityEncoding.CANONICALIZED_CODE_COLUMN); final Column targetCanonicalizedCode = targetContext.getField( QuantityEncoding.CANONICALIZED_CODE_COLUMN); + final Column resultCode = getResultUnit(operation, sourceCanonicalizedCode, + targetCanonicalizedCode); + final Column resultStruct = QuantityEncoding.toStruct( sourceContext.getField("id"), resultColumn, @@ -96,33 +167,31 @@ public static Function buildMathOperation(@Nonnull fina // The only Quantities that are decoded are calendar duration quantities parsed from literals. lit(null), sourceContext.getField("comparator"), - sourceCanonicalizedCode, + resultCode, sourceContext.getField("system"), - sourceCanonicalizedCode, + resultCode, resultColumn, - sourceCanonicalizedCode, + resultCode, sourceContext.getField("_fid") ); - final Column resultQuantityColumn = - when(sourceCanonicalizedCode.equalTo(targetCanonicalizedCode), resultStruct) - .otherwise(null); + + final Column validResult = getValidResult(operation, resultStruct, + sourceCanonicalizedCode, targetCanonicalizedCode); + final Column resultQuantityColumn = when(sourceContext.isNull().or(targetContext.isNull()), + null).otherwise(validResult); final Column idColumn = source.getIdColumn(); final Optional eidColumn = findEidColumn(source, target); final Optional thisColumn = findThisColumn(source, target); - - switch (operation) { - case ADDITION: - case SUBTRACTION: - case MULTIPLICATION: - case DIVISION: - return ElementPath + return + elementDefinition.map(definition -> ElementPath + .build(expression, dataset, idColumn, eidColumn, resultQuantityColumn, true, + Optional.empty(), + thisColumn, definition)).orElseGet(() -> ElementPath .build(expression, dataset, idColumn, eidColumn, resultQuantityColumn, true, Optional.empty(), - thisColumn, fhirType); - default: - throw new AssertionError("Unsupported math operation encountered: " + operation); - } + thisColumn, FHIRDefinedType.QUANTITY)); + }; } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 9ee9a81e0f..87c02db035 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -48,29 +48,58 @@ private QuantityEncoding() { .put("milliseconds", "ms") .build(); - public static final String CANONICALIZED_VALUE_COLUMN = QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME(); - public static final String CANONICALIZED_CODE_COLUMN = QuantitySupport.CODE_CANONICALIZED_FIELD_NAME(); + public static final String CANONICALIZED_VALUE_COLUMN = QuantitySupport + .VALUE_CANONICALIZED_FIELD_NAME(); + public static final String CANONICALIZED_CODE_COLUMN = QuantitySupport + .CODE_CANONICALIZED_FIELD_NAME(); /** * Encodes a Quantity to a Row (spark SQL compatible type) * * @param quantity a coding to encode + * @param includeScale whether the scale of the value should be encoded (or set to null) * @return the Row representation of the quantity */ @Nullable - public static Row encode(@Nullable final Quantity quantity) { + public static Row encode(@Nullable final Quantity quantity, boolean includeScale) { if (quantity == null) { return null; } + final BigDecimal value = quantity.getValue(); + @Nullable final String code = quantity.getCode(); + final String canonicalizedValue; + final String canonicalizedCode; + if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { + canonicalizedValue = Ucum.getCanonicalValueAsString(value, code); + canonicalizedCode = Ucum.getCanonicalCode(value, code); + } else { + canonicalizedValue = null; + canonicalizedCode = null; + } final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); - if (quantity.getValue().scale() > DecimalCustomCoder.scale()) { - quantity.setValue( - quantity.getValue().setScale(DecimalCustomCoder.scale(), RoundingMode.HALF_UP)); - } - return RowFactory.create(quantity.getId(), quantity.getValue(), quantity.getValue().scale(), - comparator, quantity.getUnit(), quantity.getSystem(), quantity.getCode(), null /* _fid */); + // TODO: The null scale support it a temporary measure becasue we currently + // cannot encode the scale of the results of arithmetic operations. + return RowFactory.create(quantity.getId(), + quantity.getValue(), + includeScale + ? quantity.getValue().scale() + : null, + comparator, + quantity.getUnit(), quantity.getSystem(), quantity.getCode(), canonicalizedValue, + canonicalizedCode, null /* _fid */); + } + + /** + * Encodes a Quantity to a Row (spark SQL compatible type) + * + * @param quantity a coding to encode + * @return the Row representation of the quantity + */ + @Nullable + public static Row encode(@Nullable final Quantity quantity) { + return encode(quantity, true); } /** @@ -124,9 +153,9 @@ public static StructType dataType() { final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); final StructField system = new StructField("system", DataTypes.StringType, true, metadata); final StructField code = new StructField("code", DataTypes.StringType, true, metadata); + // TODO: FlexDecimal final StructField canonicalizedValue = new StructField(CANONICALIZED_VALUE_COLUMN, - DataTypes.createDecimalType( - DecimalCustomCoder.precision(), DecimalCustomCoder.scale()), true, metadata); + DataTypes.StringType, true, metadata); final StructField canonicalizedCode = new StructField(CANONICALIZED_CODE_COLUMN, DataTypes.StringType, true, metadata); final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, @@ -166,7 +195,7 @@ public static Column toStruct( ) { return struct( id.as("id"), - value.as("value"), + value.cast(DecimalCustomCoder.decimalType()).as("value"), value_scale.as("value_scale"), comparator.as("comparator"), unit.as("unit"), @@ -189,18 +218,19 @@ public static Column encodeLiteral(@Nonnull final Quantity quantity) { final Optional comparator = Optional.ofNullable(quantity.getComparator()); final BigDecimal value = quantity.getValue(); - final BigDecimal canonicalizedValue; + // FlexDecima; + final String canonicalizedValue; final String canonicalizedCode; if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. - canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); + canonicalizedValue = Ucum.getCanonicalValueAsString(value, quantity.getCode()); canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); } else if (CalendarDurationUtils.isCalendarDuration(quantity) && CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the // UCUM library to canonicalize the value and code. final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); - canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); + canonicalizedValue = Ucum.getCanonicalValueAsString(value, resolvedCode); canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); } else { // If it is neither a UCUM Quantity nor a calendar duration, it will not have a canonicalized diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java index c94f28e8d8..c0f831df9a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/literal/QuantityLiteralPath.java @@ -175,7 +175,7 @@ public FHIRDefinedType getFhirType() { public Function getMathOperation(@Nonnull final MathOperation operation, @Nonnull final String expression, @Nonnull final Dataset dataset) { return QuantityPath.buildMathOperation(this, operation, expression, dataset, - FHIRDefinedType.QUANTITY); + Optional.empty()); } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java index f185da85af..80f1600176 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java @@ -8,6 +8,7 @@ import static au.csiro.pathling.test.assertions.Assertions.assertThat; import static au.csiro.pathling.test.helpers.SparkHelpers.quantityStructType; +import static au.csiro.pathling.test.helpers.SparkHelpers.rowForUcumQuantity; import static au.csiro.pathling.test.helpers.SparkHelpers.rowFromQuantity; import au.csiro.pathling.fhirpath.FhirPath; @@ -58,6 +59,7 @@ public class EqualityOperatorQuantityTest { QuantityLiteralPath calendarDurationLiteral1; QuantityLiteralPath calendarDurationLiteral2; QuantityLiteralPath calendarDurationLiteral3; + QuantityLiteralPath ucumQuantityLiteralNoUnit; @BeforeEach void setUp() { @@ -109,6 +111,12 @@ void setUp() { quantity8.setSystem(TestHelpers.UCUM_URL); quantity8.setCode("ms"); + final Quantity nonUcumQuantity = new Quantity(); + nonUcumQuantity.setValue(15); + nonUcumQuantity.setUnit("mSv"); + nonUcumQuantity.setSystem(TestHelpers.SNOMED_URL); + nonUcumQuantity.setCode("282250007"); + final Dataset leftDataset = new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) .withStructTypeColumns(quantityStructType()) @@ -121,6 +129,9 @@ void setUp() { .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-a", rowForUcumQuantity("1000", "mmol")) // 1000 mmol + .withRow("patient-b", rowForUcumQuantity("49", "%")) // 50 % + .withRow("patient-c", rowFromQuantity(nonUcumQuantity)) // non-ucum .buildWithStructValue(); left = new ElementPathBuilder(spark) .fhirType(FHIRDefinedType.QUANTITY) @@ -141,6 +152,9 @@ void setUp() { .withRow("patient-7", rowFromQuantity(quantity6)) // 30 d .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms + .withRow("patient-a", rowForUcumQuantity("1", "mol")) // 1 mol + .withRow("patient-b", rowForUcumQuantity("0.5", "1")) // 0.5 '1' + .withRow("patient-c", rowForUcumQuantity("1", "%")) // 1 % .buildWithStructValue(); right = new ElementPathBuilder(spark) .fhirType(FHIRDefinedType.QUANTITY) @@ -152,6 +166,8 @@ void setUp() { ucumQuantityLiteral1 = QuantityLiteralPath.fromUcumString("500 'mg'", left, ucumService); ucumQuantityLiteral2 = QuantityLiteralPath.fromUcumString("0.5 'g'", left, ucumService); ucumQuantityLiteral3 = QuantityLiteralPath.fromUcumString("1.8 'm'", left, ucumService); + ucumQuantityLiteralNoUnit = QuantityLiteralPath.fromUcumString("0.49 '1'", left, ucumService); + calendarDurationLiteral1 = QuantityLiteralPath.fromCalendarDurationString("30 days", left); calendarDurationLiteral2 = QuantityLiteralPath.fromCalendarDurationString("60 seconds", left); calendarDurationLiteral3 = QuantityLiteralPath.fromCalendarDurationString("1000 milliseconds", @@ -177,7 +193,10 @@ void equals() { RowFactory.create("patient-6", null), // 500 mg = {} RowFactory.create("patient-7", true), // 30 d = 30 d RowFactory.create("patient-8", true), // 60 s = 60 s - RowFactory.create("patient-9", true) // 1000 ms = 1000 ms + RowFactory.create("patient-9", true), // 1000 ms = 1000 ms + RowFactory.create("patient-a", true), // 1000 mmol = 1 mol + RowFactory.create("patient-b", false), // 49 % = 0.5 '' + RowFactory.create("patient-c", null) // non-ucum = 1 % ); } @@ -196,7 +215,10 @@ void notEquals() { RowFactory.create("patient-6", null), // 500 mg != {} RowFactory.create("patient-7", false), // 30 d != 30 d RowFactory.create("patient-8", false), // 60 s != 60 s - RowFactory.create("patient-9", false) // 1000 ms != 1000 ms + RowFactory.create("patient-9", false), // 1000 ms != 1000 ms + RowFactory.create("patient-a", false), // 1000 mmol != 1 mol + RowFactory.create("patient-b", true), // 49 % != 0.5 '' + RowFactory.create("patient-c", null) // non-ucum != 1 % ); } @@ -215,7 +237,10 @@ void ucumLiteralEquals() { RowFactory.create("patient-6", true), // 500 mg = 500 mg RowFactory.create("patient-7", null), // 30 d = 500 mg RowFactory.create("patient-8", null), // 60 s = 500 mg - RowFactory.create("patient-9", null) // 1000 ms = 500 mg + RowFactory.create("patient-9", null), // 1000 ms = 500 mg + RowFactory.create("patient-a", null), // 1000 mmol = 500 mg + RowFactory.create("patient-b", null), // 49 % = 500 mg + RowFactory.create("patient-c", null) // non-ucum = 500 mg ); input = new OperatorInput(parserContext, left, ucumQuantityLiteral2); @@ -230,7 +255,10 @@ void ucumLiteralEquals() { RowFactory.create("patient-6", true), // 500 mg = 0.5 mg RowFactory.create("patient-7", null), // 30 d = 0.5 mg RowFactory.create("patient-8", null), // 60 s = 0.5 mg - RowFactory.create("patient-9", null) // 1000 ms = 0.5 mg + RowFactory.create("patient-9", null), // 1000 ms = 0.5 mg + RowFactory.create("patient-a", null), // 1000 mmol = 0.5 mg + RowFactory.create("patient-b", null), // 49 % != = 0.5 mg + RowFactory.create("patient-c", null) // non-ucum != 0.5 mg ); input = new OperatorInput(parserContext, left, ucumQuantityLiteral3); @@ -245,7 +273,28 @@ void ucumLiteralEquals() { RowFactory.create("patient-6", null), // 500 mg = 1.8 m RowFactory.create("patient-7", null), // 30 d = 1.8 m RowFactory.create("patient-8", null), // 60 s = 1.8 m - RowFactory.create("patient-9", null) // 1000 ms = 1.8 m + RowFactory.create("patient-9", null), // 1000 ms = 1.8 m + RowFactory.create("patient-a", null), // 1000 mmol = 1.8 m + RowFactory.create("patient-b", null), // 49 % != = 1.8 m + RowFactory.create("patient-c", null) // non-ucum != 1.8 m + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteralNoUnit); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg = 1.8 m + RowFactory.create("patient-2", null), // 500 mg = 1.8 m + RowFactory.create("patient-3", null), // 500 mg = 1.8 m + RowFactory.create("patient-4", null), // 650 mg = 1.8 m + RowFactory.create("patient-5", null), // {} = 1.8 m + RowFactory.create("patient-6", null), // 500 mg = 1.8 m + RowFactory.create("patient-7", null), // 30 d = 1.8 m + RowFactory.create("patient-8", null), // 60 s = 1.8 m + RowFactory.create("patient-9", null), // 1000 ms = 1.8 m + RowFactory.create("patient-a", false), // 1000 mmol = 1.8 m + RowFactory.create("patient-b", true), // 49 % != = 1.8 m + RowFactory.create("patient-c", null) // non-ucum != 1.8 m ); input = new OperatorInput(parserContext, ucumQuantityLiteral1, ucumQuantityLiteral1); @@ -256,7 +305,7 @@ void ucumLiteralEquals() { .withColumn(DataTypes.BooleanType) .withIdsAndValue(true, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", - "patient-7", "patient-8", "patient-9")) + "patient-7", "patient-8", "patient-9", "patient-a", "patient-b", "patient-c")) .build(); assertThat(result).selectOrderedResult().hasRows(allTrue); } @@ -276,7 +325,10 @@ void ucumLiteralNotEquals() { RowFactory.create("patient-6", false), // 500 mg != 500 mg RowFactory.create("patient-7", null), // 30 d != 500 mg RowFactory.create("patient-8", null), // 60 s != 500 mg - RowFactory.create("patient-9", null) // 1000 ms != 500 mg + RowFactory.create("patient-9", null), // 1000 ms != 500 mg + RowFactory.create("patient-a", null), // 1000 mmol != 500 mg + RowFactory.create("patient-b", null), // 49 % != 500 mg + RowFactory.create("patient-c", null) // non-ucum != 500 mg ); input = new OperatorInput(parserContext, left, ucumQuantityLiteral2); @@ -291,7 +343,10 @@ void ucumLiteralNotEquals() { RowFactory.create("patient-6", false), // 500 mg != 0.5 mg RowFactory.create("patient-7", null), // 30 d != 0.5 mg RowFactory.create("patient-8", null), // 60 s != 0.5 mg - RowFactory.create("patient-9", null) // 1000 ms != 0.5 mg + RowFactory.create("patient-9", null), // 1000 ms != 0.5 mg + RowFactory.create("patient-a", null), // 1000 mmol != 0.5 mg + RowFactory.create("patient-b", null), // 49 % != 0.5 mg + RowFactory.create("patient-c", null) // non-ucum != 0.5 mg ); input = new OperatorInput(parserContext, left, ucumQuantityLiteral3); @@ -306,7 +361,28 @@ void ucumLiteralNotEquals() { RowFactory.create("patient-6", null), // 500 mg != 1.8 m RowFactory.create("patient-7", null), // 30 d != 1.8 m RowFactory.create("patient-8", null), // 60 s != 1.8 m - RowFactory.create("patient-9", null) // 1000 ms != 1.8 m + RowFactory.create("patient-9", null), // 1000 ms != 1.8 m + RowFactory.create("patient-a", null), // 1000 mmol != 1.8 m + RowFactory.create("patient-b", null), // 49 % != 1.8 m + RowFactory.create("patient-c", null) // non-ucum != 1.8 m + ); + + input = new OperatorInput(parserContext, left, ucumQuantityLiteralNoUnit); + result = equalityOperator.invoke(input); + + assertThat(result).selectOrderedResult().hasRows( + RowFactory.create("patient-1", null), // 500 mg != 0.49 '' + RowFactory.create("patient-2", null), // 500 mg != 0.49 '' + RowFactory.create("patient-3", null), // 500 mg != 0.49 '' + RowFactory.create("patient-4", null), // 650 mg != 0.49 '' + RowFactory.create("patient-5", null), // {} != 0.49 '' + RowFactory.create("patient-6", null), // 500 mg != 0.49 '' + RowFactory.create("patient-7", null), // 30 d != 0.49 '' + RowFactory.create("patient-8", null), // 60 s != 0.49 '' + RowFactory.create("patient-9", null), // 1000 ms != 0.49 '' + RowFactory.create("patient-a", true), // 1000 mmol != 0.49 '' + RowFactory.create("patient-b", false), // 49 % != 0.49 '' + RowFactory.create("patient-c", null) // non-ucum != 0.49 '' ); input = new OperatorInput(parserContext, ucumQuantityLiteral1, ucumQuantityLiteral1); @@ -317,7 +393,7 @@ void ucumLiteralNotEquals() { .withColumn(DataTypes.BooleanType) .withIdsAndValue(false, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", - "patient-7", "patient-8", "patient-9")) + "patient-7", "patient-8", "patient-9", "patient-a", "patient-b", "patient-c")) .build(); assertThat(result).selectOrderedResult().hasRows(allFalse); } @@ -337,7 +413,10 @@ void calendarLiteralEquals() { RowFactory.create("patient-6", null), // 500 mg = 30 days RowFactory.create("patient-7", null), // 30 d = 30 days RowFactory.create("patient-8", null), // 60 s = 30 days - RowFactory.create("patient-9", null) // 1000 ms = 30 days + RowFactory.create("patient-9", null), // 1000 ms = 30 days + RowFactory.create("patient-a", null), // 1000 mmol = 30 days + RowFactory.create("patient-b", null), // 49 % = 30 days + RowFactory.create("patient-c", null) // non-ucum = 30 days ); input = new OperatorInput(parserContext, left, calendarDurationLiteral2); @@ -352,7 +431,10 @@ void calendarLiteralEquals() { RowFactory.create("patient-6", null), // 500 mg = 60 seconds RowFactory.create("patient-7", false), // 30 d = 60 seconds RowFactory.create("patient-8", true), // 60 s = 60 seconds - RowFactory.create("patient-9", false) // 1000 ms = 60 seconds + RowFactory.create("patient-9", false), // 1000 ms = 60 seconds + RowFactory.create("patient-a", null), // 1000 mmol = 60 seconds + RowFactory.create("patient-b", null), // 49 % = 60 seconds + RowFactory.create("patient-c", null) // non-ucum = 60 seconds ); input = new OperatorInput(parserContext, left, calendarDurationLiteral3); @@ -367,7 +449,10 @@ void calendarLiteralEquals() { RowFactory.create("patient-6", null), // 500 mg = 1000 milliseconds RowFactory.create("patient-7", false), // 30 d = 1000 milliseconds RowFactory.create("patient-8", false), // 60 s = 1000 milliseconds - RowFactory.create("patient-9", true) // 1000 ms = 1000 milliseconds + RowFactory.create("patient-9", true), // 1000 ms = 1000 milliseconds + RowFactory.create("patient-a", null), // 1000 mmol = 1000 milliseconds + RowFactory.create("patient-b", null), // 49 % = 1000 milliseconds + RowFactory.create("patient-c", null) // non-ucum = 1000 milliseconds ); input = new OperatorInput(parserContext, calendarDurationLiteral2, calendarDurationLiteral2); @@ -378,7 +463,7 @@ void calendarLiteralEquals() { .withColumn(DataTypes.BooleanType) .withIdsAndValue(true, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", - "patient-7", "patient-8", "patient-9")) + "patient-7", "patient-8", "patient-9", "patient-a", "patient-b", "patient-c")) .build(); assertThat(result).selectOrderedResult().hasRows(allTrue); } @@ -398,7 +483,10 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-6", null), // 500 mg != 30 days RowFactory.create("patient-7", null), // 30 d != 30 days RowFactory.create("patient-8", null), // 60 s != 30 days - RowFactory.create("patient-9", null) // 1000 ms != 30 days + RowFactory.create("patient-9", null), // 1000 ms != 30 days + RowFactory.create("patient-a", null), // 1000 mmol != 30 days + RowFactory.create("patient-b", null), // 49 % != 30 days + RowFactory.create("patient-c", null) // non-ucum != 30 days ); input = new OperatorInput(parserContext, left, calendarDurationLiteral2); @@ -413,7 +501,10 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-6", null), // 500 mg != 60 s RowFactory.create("patient-7", true), // 30 d != 60 s RowFactory.create("patient-8", false), // 60 s != 60 s - RowFactory.create("patient-9", true) // 1000 ms != 60 s + RowFactory.create("patient-9", true), // 1000 ms != 60 s + RowFactory.create("patient-a", null), // 1000 mmol != 60 seconds + RowFactory.create("patient-b", null), // 49 % != 60 seconds + RowFactory.create("patient-c", null) // non-ucum != 60 seconds ); input = new OperatorInput(parserContext, left, calendarDurationLiteral3); @@ -428,7 +519,10 @@ void calendarLiteralNotEquals() { RowFactory.create("patient-6", null), // 500 mg != 1000 ms RowFactory.create("patient-7", true), // 30 d != 1000 ms RowFactory.create("patient-8", true), // 60 s != 1000 ms - RowFactory.create("patient-9", false) // 1000 ms != 1000 ms + RowFactory.create("patient-9", false), // 1000 ms != 1000 ms + RowFactory.create("patient-a", null), // 1000 mmol != 1000 milliseconds + RowFactory.create("patient-b", null), // 49 % != 1000 milliseconds + RowFactory.create("patient-c", null) // non-ucum != 1000 milliseconds ); input = new OperatorInput(parserContext, calendarDurationLiteral2, calendarDurationLiteral2); @@ -439,7 +533,7 @@ void calendarLiteralNotEquals() { .withColumn(DataTypes.BooleanType) .withIdsAndValue(false, List.of("patient-1", "patient-2", "patient-3", "patient-4", "patient-5", "patient-6", - "patient-7", "patient-8", "patient-9")) + "patient-7", "patient-8", "patient-9", "patient-a", "patient-b", "patient-c")) .build(); assertThat(result).selectOrderedResult().hasRows(allFalse); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java index c0b24943ff..f8cb31971c 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorQuantityTest.java @@ -104,32 +104,40 @@ Stream parameters() { } Dataset expectedResult(@Nonnull final String operator) { - final Row result; + final Row result1; + final Row result2; + switch (operator) { case "+": - result = rowForUcumQuantity(new BigDecimal("1650.000000"), "m"); + result1 = rowForUcumQuantity(new BigDecimal("1650.0"), "m"); + result2 = null; break; case "-": - result = rowForUcumQuantity(new BigDecimal("-1350.000000"), "m"); + result1 = rowForUcumQuantity(new BigDecimal("-1350.0"), "m"); + result2 = null; break; case "*": - result = rowForUcumQuantity(new BigDecimal("225000.000000"), "m"); + result1 = null; + result2 = rowForUcumQuantity("1000.0", "g"); break; case "/": - result = rowForUcumQuantity(new BigDecimal("0.100000"), "m"); + result1 = rowForUcumQuantity(new BigDecimal("0.1"), "1"); + result2 = rowForUcumQuantity("4000", "g"); break; default: - result = null; + result1 = null; + result2 = null; } return new DatasetBuilder(spark) .withIdColumn(ID_ALIAS) .withStructTypeColumns(quantityStructType()) - .withRow("patient-1", result) - .withRow("patient-2", null) + .withRow("patient-1", result1) + .withRow("patient-2", result2) .withRow("patient-3", null) .withRow("patient-4", null) .withRow("patient-5", null) .withRow("patient-6", null) + .withRow("patient-7", null) .buildWithStructValue(); } @@ -148,20 +156,23 @@ FhirPath buildQuantityExpression(final boolean leftOperand) { ? rowForUcumQuantity(new BigDecimal("150.0"), "m") : rowForUcumQuantity(new BigDecimal("1.5"), "km")) .withRow("patient-2", leftOperand + ? rowForUcumQuantity(new BigDecimal("2.0"), "kg") + : rowForUcumQuantity(new BigDecimal("0.5"), "1")) + .withRow("patient-3", leftOperand ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") : rowForUcumQuantity(new BigDecimal("1.5"), "h")) // Not comparable - .withRow("patient-3", leftOperand + .withRow("patient-4", leftOperand ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") : rowFromQuantity(nonUcumQuantity)) // Not comparable - .withRow("patient-4", leftOperand + .withRow("patient-5", leftOperand ? null : rowForUcumQuantity(new BigDecimal("1.5"), "h")) - .withRow("patient-5", leftOperand + .withRow("patient-6", leftOperand ? rowForUcumQuantity(new BigDecimal("7.7"), "mSv") : null) - .withRow("patient-6", null) + .withRow("patient-7", null) .buildWithStructValue(); return new ElementPathBuilder(spark) .fhirType(FHIRDefinedType.QUANTITY) @@ -211,7 +222,7 @@ void volumeArithmetic() { .withIdColumn() .withStructTypeColumns(quantityStructType()) .withRow("patient-1", - rowForUcumQuantity(new BigDecimal("1204427340000000000000000.00"), "m-3")) + rowForUcumQuantity(new BigDecimal("1204427340000000000000000"), "m-3")) .buildWithStructValue(); assertThat(result).selectOrderedResult().hasRows(expectedDataset); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java index 73b907046c..0d7c01b5e4 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java @@ -60,9 +60,9 @@ public class QuantityOperatorsPrecisionTest { // (if available) static final String REASONABLE_DECIMAL_01 = createSpanningDecimal(9, 3, 1, - 6).toString(); // 9000.00001 + 6).toString(); // 9000.000001 static final String REASONABLE_DECIMAL_02 = createSpanningDecimal(9, 3, 2, - 6).toString(); // 9000.00002 + 6).toString(); // 9000.000002 // for Decimal(32,6) static final String FULL_DECIMAL_01 = createSpanningDecimal(9, 26, 1, diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java index 87451f695b..a27d1a9a40 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java @@ -32,6 +32,7 @@ import au.csiro.pathling.fhirpath.element.DatePath; import au.csiro.pathling.fhirpath.element.DecimalPath; import au.csiro.pathling.fhirpath.element.IntegerPath; +import au.csiro.pathling.fhirpath.element.QuantityPath; import au.csiro.pathling.fhirpath.element.StringPath; import au.csiro.pathling.fhirpath.encoding.SimpleCoding; import au.csiro.pathling.fhirpath.literal.CodingLiteralPath; @@ -759,5 +760,40 @@ private void setSubjectResource(@Nonnull final ResourceType resourceType) { .build(); parser = new Parser(parserContext); } + + @Test + void testQuantityMultiplicationAndDivision() { + assertThatResultOf( + "((reverseResolve(Observation.subject).where(valueQuantity < 150 'cm').valueQuantity.first() * 2 '1')" + + " / reverseResolve(Observation.subject).where(valueQuantity < 1.50 'm').valueQuantity.first()).value") + .isElementPath(DecimalPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/testQuantityMultiplicationAndDivision.csv"); + } + + @Test + void testQuantityAdditionSubtractionAndEquality() { + // 33 'mmol/L == 19873051110000000000000000 'm-3' + assertThatResultOf( + "((reverseResolve(Observation.subject).where(valueQuantity > 1 'mmol/L').valueQuantity.first() + 33 'mmol/L')" + + " - reverseResolve(Observation.subject).where(valueQuantity > 1 'mmol/L').valueQuantity.first()) = 19873051110000000000000000 'm-3'") + .isElementPath(BooleanPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/testQuantityAdditionSubtractionAndEquality.csv"); + } + @Test + void testQuantityAdditionWithOverflow() { + // values for 121503c8-9564-4b48-9086-a22df717948e and a7eb2ce7-1075-426c-addd-957b861b0e55 exceed 10^26 m-3 + assertThatResultOf( + "(reverseResolve(Observation.subject).where(valueQuantity > 100 'mmol/L').valueQuantity.first() + 33 'mmol/L').value") + .isElementPath(DecimalPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/testQuantityAdditionWithOverflow_value.csv"); + assertThatResultOf( + "(reverseResolve(Observation.subject).where(valueQuantity > 100 'mmol/L').valueQuantity.first() + 33 'mmol/L').code") + .isElementPath(StringPath.class) + .selectResult() + .hasRows(spark, "responses/ParserTest/testQuantityAdditionWithOverflow_code.csv"); + } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java index f2a439410d..3cefd4196c 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -1,28 +1,15 @@ package au.csiro.pathling.test.benchmark; -import au.csiro.pathling.Configuration; -import au.csiro.pathling.aggregate.AggregateExecutor; -import au.csiro.pathling.aggregate.AggregateRequest; -import au.csiro.pathling.aggregate.AggregateRequestBuilder; -import au.csiro.pathling.aggregate.AggregateResponse; -import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; -import au.csiro.pathling.fhir.TerminologyServiceFactory; -import au.csiro.pathling.io.Database; import au.csiro.pathling.jmh.AbstractJmhSpringBootState; import au.csiro.pathling.sql.types.FlexDecimal; +import au.csiro.pathling.sql.types.FlexiDecimal; import au.csiro.pathling.sql.types.UcumDecimal; -import au.csiro.pathling.terminology.TerminologyService; -import au.csiro.pathling.test.SharedMocks; import au.csiro.pathling.test.builders.DatasetBuilder; -import au.csiro.pathling.test.helpers.TestHelpers; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.parser.IParser; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; -import org.hl7.fhir.r4.model.Enumerations.ResourceType; import org.junit.jupiter.api.Tag; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; @@ -31,7 +18,6 @@ import javax.annotation.Nonnull; import java.math.BigDecimal; import java.util.List; -import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.mockito.Mockito.mock; @@ -45,8 +31,8 @@ public class DecimalBenchmark { private static final int ROWS = 1000000; - private static final BigDecimal LEFT_DECIMAL = new BigDecimal("1493493840938434300000.123456"); - private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("999999999999138409380.123456"); + private static final BigDecimal LEFT_DECIMAL = new BigDecimal("12345678901234567890123456.123456"); + private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("0.12345678901234567890123456"); @State(Scope.Benchmark) @ActiveProfiles("unit-test") @@ -66,6 +52,8 @@ public void setUp() { .withColumn("rightDecimal", DecimalCustomCoder.decimalType()) .withColumn("leftFlexDecimal", FlexDecimal.DATA_TYPE) .withColumn("rightFlexDecimal", FlexDecimal.DATA_TYPE) + .withColumn("leftFlexiDecimal", FlexiDecimal.DATA_TYPE) + .withColumn("rightFlexiDecimal", FlexiDecimal.DATA_TYPE) .withColumn("leftUcumDecimal", UcumDecimal.DATA_TYPE) .withColumn("rightUcumDecimal", UcumDecimal.DATA_TYPE); @@ -73,8 +61,10 @@ public void setUp() { datasetBuilder = datasetBuilder.withRow( LEFT_DECIMAL, RIGHT_DECIMAL, - FlexDecimal.toRow(LEFT_DECIMAL), - FlexDecimal.toRow(RIGHT_DECIMAL), + FlexDecimal.toValue(LEFT_DECIMAL), + FlexDecimal.toValue(RIGHT_DECIMAL), + FlexiDecimal.toValue(LEFT_DECIMAL), + FlexiDecimal.toValue(RIGHT_DECIMAL), LEFT_DECIMAL.toString(), RIGHT_DECIMAL.toString() ); @@ -145,7 +135,33 @@ public void lt_flexDec_Benchmark(final Blackhole bh, bh.consume(ds.collectQuery( FlexDecimal.lt(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); } + @Benchmark + public void multiply_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.multiply(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + + @Benchmark + public void add_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.plus(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + @Benchmark + public void equals_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.equals(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + + @Benchmark + public void lt_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.lt(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } @Benchmark public void multiply_ucumDec_Benchmark(final Blackhole bh, diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java index 89594f5715..a4600cedeb 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/helpers/SparkHelpers.java @@ -21,10 +21,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import au.csiro.pathling.utilities.Preconditions; import lombok.Value; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Metadata; @@ -151,24 +154,7 @@ public static Row rowFromCodeableConcept(@Nonnull final CodeableConcept codeable @Nonnull public static Row rowFromQuantity(@Nonnull final Quantity quantity) { - final BigDecimal value = quantity.getValue(); - final String code = quantity.getCode(); - final BigDecimal canonicalizedValue; - final String canonicalizedCode; - if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { - canonicalizedValue = Ucum.getCanonicalValue(value, code); - canonicalizedCode = Ucum.getCanonicalCode(value, code); - } else { - canonicalizedValue = null; - canonicalizedCode = null; - } - final String comparator = Optional.ofNullable(quantity.getComparator()) - .map(QuantityComparator::toCode).orElse(null); - return new GenericRowWithSchema( - new Object[]{quantity.getId(), quantity.getValue(), null /* scale */, comparator, - quantity.getUnit(), quantity.getSystem(), quantity.getCode(), canonicalizedValue, - canonicalizedCode, null /* _fid */}, - quantityStructType()); + return Preconditions.checkNotNull(QuantityEncoding.encode(quantity, false)); } @Nonnull @@ -180,12 +166,12 @@ public static Row rowForUcumQuantity(@Nonnull final BigDecimal value, quantity.setValue( quantity.getValue().setScale(DecimalCustomCoder.scale(), RoundingMode.HALF_UP)); } - // TODO: This is not really how it should work - // As the BigDecimal::precision is not same as DecimalType precision(). - // Besides this results in the value being set to null - // if (quantity.getValue().precision() > DecimalCustomCoder.precision()) { - // throw new AssertionError("Attempt to encode a value with greater than supported precision"); - // } + // NOTE: BigDecimal precision is total number od digits (before and after the decimal point) + // while the SQL decimal precision is the number of digits allowed before the decimal point. + if (quantity.getValue().precision() + > DecimalCustomCoder.precision() + DecimalCustomCoder.scale()) { + throw new AssertionError("Attempt to encode a value with greater than supported precision"); + } quantity.setUnit(unit); quantity.setSystem(TestHelpers.UCUM_URL); quantity.setCode(unit); diff --git a/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionSubtractionAndEquality.csv b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionSubtractionAndEquality.csv new file mode 100644 index 0000000000..3079631f8e --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionSubtractionAndEquality.csv @@ -0,0 +1,9 @@ +121503c8-9564-4b48-9086-a22df717948e,true +2b36c1e2-bbe1-45ae-8124-4adad2677702, +7001ad9c-34d2-4eb5-8165-5fdc2147f469, +8ee183e2-b3c0-4151-be94-b945d6aa8c6d,true +9360820c-8602-4335-8b50-c88d627a0c20,true +a7eb2ce7-1075-426c-addd-957b861b0e55,true +bbd33563-70d9-4f6d-a79a-dd1fc55f5ad9, +beff242e-580b-47c0-9844-c1a68c36c5bf, +e62e52ae-2d75-4070-a0ae-3cc78d35ed08,true diff --git a/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_code.csv b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_code.csv new file mode 100644 index 0000000000..6f23efbb9d --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_code.csv @@ -0,0 +1,9 @@ +121503c8-9564-4b48-9086-a22df717948e,m-3 +2b36c1e2-bbe1-45ae-8124-4adad2677702, +7001ad9c-34d2-4eb5-8165-5fdc2147f469, +8ee183e2-b3c0-4151-be94-b945d6aa8c6d,m-3 +9360820c-8602-4335-8b50-c88d627a0c20,m-3 +a7eb2ce7-1075-426c-addd-957b861b0e55,m-3 +bbd33563-70d9-4f6d-a79a-dd1fc55f5ad9, +beff242e-580b-47c0-9844-c1a68c36c5bf, +e62e52ae-2d75-4070-a0ae-3cc78d35ed08,m-3 diff --git a/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_value.csv b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_value.csv new file mode 100644 index 0000000000..d804ee2487 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/testQuantityAdditionWithOverflow_value.csv @@ -0,0 +1,9 @@ +121503c8-9564-4b48-9086-a22df717948e, +2b36c1e2-bbe1-45ae-8124-4adad2677702, +7001ad9c-34d2-4eb5-8165-5fdc2147f469, +8ee183e2-b3c0-4151-be94-b945d6aa8c6d,85389546411730050624531900 +9360820c-8602-4335-8b50-c88d627a0c20,83016978156556952635728000 +a7eb2ce7-1075-426c-addd-957b861b0e55, +bbd33563-70d9-4f6d-a79a-dd1fc55f5ad9, +beff242e-580b-47c0-9844-c1a68c36c5bf, +e62e52ae-2d75-4070-a0ae-3cc78d35ed08,85425566063603919179553900 diff --git a/fhir-server/src/test/resources/responses/ParserTest/testQuantityMultiplicationAndDivision.csv b/fhir-server/src/test/resources/responses/ParserTest/testQuantityMultiplicationAndDivision.csv new file mode 100644 index 0000000000..08b66438c0 --- /dev/null +++ b/fhir-server/src/test/resources/responses/ParserTest/testQuantityMultiplicationAndDivision.csv @@ -0,0 +1,9 @@ +121503c8-9564-4b48-9086-a22df717948e,2.000000 +2b36c1e2-bbe1-45ae-8124-4adad2677702,2.000000 +7001ad9c-34d2-4eb5-8165-5fdc2147f469, +8ee183e2-b3c0-4151-be94-b945d6aa8c6d, +9360820c-8602-4335-8b50-c88d627a0c20, +a7eb2ce7-1075-426c-addd-957b861b0e55, +bbd33563-70d9-4f6d-a79a-dd1fc55f5ad9,2.000000 +beff242e-580b-47c0-9844-c1a68c36c5bf, +e62e52ae-2d75-4070-a0ae-3cc78d35ed08, From 428f990276fc16d2b49b31935cd212f0c1bb293a Mon Sep 17 00:00:00 2001 From: Piotr Szul Date: Wed, 12 Oct 2022 14:20:47 +0200 Subject: [PATCH 63/83] Switched canonicalized value to FlexiDecimal based on struct(Decimal, int) --- .../encoders/terminology/ucum/Ucum.java | 22 +-- .../pathling/sql/types/FlexiDecimal.java | 139 ++++++++++++++++-- .../csiro/pathling/sql/types/UcumDecimal.java | 77 ---------- .../pathling/encoders/QuantitySupport.scala | 34 ++--- .../sql/types/FlexiDecimalSupport.scala | 54 +++++++ .../encoders/LightweightFhirEncodersTest.java | 8 +- .../encoders/SchemaConverterTest.java | 8 +- .../pathling/sql/types/FlexiDecimalTest.java | 32 ++++ .../comparison/QuantitySqlComparator.java | 12 +- .../fhirpath/element/QuantityPath.java | 12 +- .../fhirpath/encoding/QuantityEncoding.java | 23 +-- .../EqualityOperatorQuantityTest.java | 38 ++--- .../QuantityOperatorsPrecisionTest.java | 37 ++++- .../fhirpath/parser/AbstractParserTest.java | 13 ++ .../parser/DateTimeArithmeticParserTest.java | 2 +- .../pathling/fhirpath/parser/ParserTest.java | 12 +- .../fhirpath/parser/QuantityParserTest.java | 2 +- .../test/benchmark/DecimalBenchmark.java | 40 +---- 18 files changed, 327 insertions(+), 238 deletions(-) delete mode 100644 encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java create mode 100644 encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala create mode 100644 encoders/src/test/java/au/csiro/pathling/sql/types/FlexiDecimalTest.java diff --git a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java index 7f1fc0b375..8d17d1cfa5 100644 --- a/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java +++ b/encoders/src/main/java/au/csiro/pathling/encoders/terminology/ucum/Ucum.java @@ -16,13 +16,16 @@ import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import java.io.InputStream; import java.math.BigDecimal; +import java.math.RoundingMode; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import au.csiro.pathling.sql.types.FlexiDecimal; import org.fhir.ucum.Decimal; import org.fhir.ucum.Pair; import org.fhir.ucum.UcumEssenceService; import org.fhir.ucum.UcumException; import org.fhir.ucum.UcumService; +import org.hl7.fhir.r4.model.DecimalType; /** * Makes UCUM services available to the rest of the application. @@ -74,25 +77,6 @@ public static BigDecimal getCanonicalValue(@Nullable final BigDecimal value, } } - @Nullable - public static String getCanonicalValueAsString(@Nullable final BigDecimal value, - @Nullable final String code) { - try { - @Nullable final Pair result = getCanonicalForm(value, code); - if (result == null) { - return null; - } - @Nullable final Decimal decimalValue = result.getValue(); - if (decimalValue == null) { - return null; - } - @Nullable final String stringValue = decimalValue.asDecimal(); - return stringValue; - } catch (final UcumException e) { - return null; - } - } - @Nullable public static String getCanonicalCode(@Nullable final BigDecimal value, @Nullable final String code) { diff --git a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java index e581ca3a55..96ee92df2e 100644 --- a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java +++ b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java @@ -1,42 +1,69 @@ package au.csiro.pathling.sql.types; +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.api.java.UDF1; import org.apache.spark.sql.api.java.UDF2; import org.apache.spark.sql.expressions.UserDefinedFunction; import org.apache.spark.sql.functions; import org.apache.spark.sql.types.*; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.math.BigDecimal; +import java.math.RoundingMode; +/** + * Implementation of flexible decimal type represented as the unscaled value with up to 38 digits + * and the scale. + */ public class FlexiDecimal { + /** + * The maximum precision (the number of significant digits) + */ + public static final int MAX_PRECISION = 38; + /** + * The Sql type of for the unscaled value. + */ + public static final DataType DECIMAL_TYPE = DataTypes.createDecimalType(MAX_PRECISION, 0); + @Nonnull private static StructType createFlexibleDecimalType() { final Metadata metadata = new MetadataBuilder().build(); - final StructField value = new StructField("value", DataTypes.createDecimalType(32, 0), true, + final StructField value = new StructField("value", DECIMAL_TYPE, true, metadata); final StructField scale = new StructField("scale", DataTypes.IntegerType, true, metadata); return new StructType(new StructField[]{value, scale}); } + /** + * The Sql (struct) type for flexible decimal + */ @Nonnull public static DataType DATA_TYPE = createFlexibleDecimalType(); @Nonnull private static UserDefinedFunction toBooleanUdf( UDF2 method) { - final UDF2 f = (left, right) -> - method.call(fromValue(left), fromValue(right)); + final UDF2 f = (left, right) -> { + final BigDecimal leftValue = fromValue(left); + final BigDecimal rightValue = fromValue(right); + return (leftValue == null || rightValue == null) + ? null + : method.call(leftValue, rightValue); + }; return functions.udf(f, DataTypes.BooleanType); } @Nonnull private static UDF2 wrapBigDecimal2( UDF2 method) { - return (left, right) -> toValue( - method.call(fromValue(left), fromValue(right))); + return (left, right) -> + (left == null || right == null) + ? null + : toValue(method.call(fromValue(left), fromValue(right))); } @Nonnull @@ -45,22 +72,78 @@ private static UserDefinedFunction toBigDecimalUdf( return functions.udf(wrapBigDecimal2(method), DATA_TYPE); } - @Nonnull - public static BigDecimal fromValue(@Nonnull final Row row) { - final BigDecimal unscaledValue = row.getDecimal(0); - return unscaledValue.movePointLeft(row.getInt(1)); + /** + * Decodes a flexible decimal from the Row + * + * @param row the row to decode + * @return the BigDecmal representation of the row + */ + @Nullable + public static BigDecimal fromValue(@Nullable final Row row) { + return row != null && !row.isNullAt(0) + ? row.getDecimal(0).movePointLeft(row.getInt(1)) + : null; } - @Nonnull - public static Row toValue(@Nonnull final BigDecimal decimal) { - return RowFactory.create(decimal.movePointRight(decimal.scale()), decimal.scale()); + /** + * Encodes a flexible decimal into a Row + * + * @param decimal the decimal to encode + * @return the Row representation of the decimal + */ + @Nullable + public static Row toValue(@Nullable final BigDecimal decimal) { + final Object[] fieldValues = toArrayValue(decimal); + return fieldValues != null + ? RowFactory.create(fieldValues) + : null; + } + + @Nullable + private static Object[] toArrayValue(@Nullable final BigDecimal decimal) { + BigDecimal normalizedValue = normalize(decimal); + return normalizedValue != null + ? new Object[]{Decimal.apply(normalizedValue.unscaledValue()), normalizedValue.scale()} + : null; + } + + @Nullable + public static BigDecimal normalize(@Nullable final BigDecimal decimal) { + if (decimal == null) { + return null; + } else { + final BigDecimal adjustedValue = decimal.scale() < 0 + ? decimal.setScale(0, RoundingMode.UNNECESSARY) + : decimal; + // This may be may have too many digits + if (adjustedValue.precision() > MAX_PRECISION) { + // we need to adjust the scale to fit into the desired precision + int desiredScale = adjustedValue.scale() - (adjustedValue.precision() - MAX_PRECISION); + if (desiredScale >= 0) { + return adjustedValue.setScale(desiredScale, RoundingMode.HALF_UP); + } else { + return null; + } + } else { + return adjustedValue; + } + } } - private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf(BigDecimal::equals); + private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf((l, r) -> l.compareTo(r) == 0); private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) < 0); + private static final UserDefinedFunction LTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) <= 0); + private static final UserDefinedFunction GT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) > 0); + private static final UserDefinedFunction GTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) >= 0); private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(BigDecimal::add); private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(BigDecimal::multiply); + private static final UserDefinedFunction MINUS_UDF = toBigDecimalUdf(BigDecimal::subtract); + private static final UserDefinedFunction DIVIDE_UDF = toBigDecimalUdf(BigDecimal::divide); + + private static final UserDefinedFunction TO_DECIMAL = functions.udf( + (UDF1) FlexiDecimal::fromValue, + DecimalCustomCoder.decimalType()); @Nonnull public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { @@ -72,6 +155,21 @@ public static Column lt(@Nonnull final Column left, @Nonnull final Column right) return LT_UDF.apply(left, right); } + @Nonnull + public static Column lte(@Nonnull final Column left, @Nonnull final Column right) { + return LTE_UDF.apply(left, right); + } + + @Nonnull + public static Column gt(@Nonnull final Column left, @Nonnull final Column right) { + return GT_UDF.apply(left, right); + } + + @Nonnull + public static Column gte(@Nonnull final Column left, @Nonnull final Column right) { + return GTE_UDF.apply(left, right); + } + @Nonnull public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { return PLUS_UDF.apply(left, right); @@ -81,4 +179,19 @@ public static Column plus(@Nonnull final Column left, @Nonnull final Column righ public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { return MULTIPLY_UDF.apply(left, right); } + + @Nonnull + public static Column minus(@Nonnull final Column left, @Nonnull final Column right) { + return MINUS_UDF.apply(left, right); + } + + @Nonnull + public static Column divide(@Nonnull final Column left, @Nonnull final Column right) { + return DIVIDE_UDF.apply(left, right); + } + + @Nonnull + public static Column to_decimal(@Nonnull final Column flexiDecimal) { + return TO_DECIMAL.apply(flexiDecimal); + } } diff --git a/encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java deleted file mode 100644 index cc4f0c4abb..0000000000 --- a/encoders/src/main/java/au/csiro/pathling/sql/types/UcumDecimal.java +++ /dev/null @@ -1,77 +0,0 @@ -package au.csiro.pathling.sql.types; - -import org.apache.spark.sql.Column; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.RowFactory; -import org.apache.spark.sql.api.java.UDF2; -import org.apache.spark.sql.expressions.UserDefinedFunction; -import org.apache.spark.sql.functions; -import org.apache.spark.sql.types.*; -import org.fhir.ucum.Decimal; -import org.fhir.ucum.UcumException; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.math.BigDecimal; - -public class UcumDecimal { - - @Nonnull - public static DataType DATA_TYPE = DataTypes.StringType; - - @Nonnull - private static UserDefinedFunction toBooleanUdf( - UDF2 method) { - final UDF2 f = (left, right) -> - method.call(fromValue(left), fromValue(right)); - return functions.udf(f, DataTypes.BooleanType); - } - - @Nonnull - private static UserDefinedFunction toBigDecimalUdf( - UDF2 method) { - - final UDF2 f = (left, right) -> - toValue(method.call(fromValue(left), fromValue(right))); - return functions.udf(f, DATA_TYPE); - } - - @Nullable - public static Decimal fromValue(@Nonnull final String value) { - try { - return new Decimal(value); - } catch (UcumException e) { - return null; - } - } - - @Nonnull - public static String toValue(@Nonnull final Decimal decimal) { - return decimal.toString(); - } - - private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf(Decimal::equals); - private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.comparesTo(r) < 0); - - private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(Decimal::add); - private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(Decimal::multiply); - - @Nonnull - public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { - return EQUALS_UDF.apply(left, right); - } - - @Nonnull - public static Column lt(@Nonnull final Column left, @Nonnull final Column right) { - return LT_UDF.apply(left, right); - } - - @Nonnull - public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { - return PLUS_UDF.apply(left, right); - } - - @Nonnull - public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { - return MULTIPLY_UDF.apply(left, right); - } -} diff --git a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala index 13b7c13b62..e87cb3b947 100644 --- a/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala +++ b/encoders/src/main/scala/au/csiro/pathling/encoders/QuantitySupport.scala @@ -13,12 +13,14 @@ package au.csiro.pathling.encoders -import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder import au.csiro.pathling.encoders.terminology.ucum.Ucum -import au.csiro.pathling.sql.types.FlexDecimal -import org.apache.spark.sql.catalyst.expressions.Expression +import au.csiro.pathling.sql.types.FlexiDecimal +import au.csiro.pathling.sql.types.FlexiDecimalSupport.createFlexiDecimalSerializer +import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} -import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType, StructField} +import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, IsNull, Literal} +import org.apache.spark.sql.functions.{lit, struct} +import org.apache.spark.sql.types.{DataTypes, Decimal, DecimalType, ObjectType, StructField} import org.apache.spark.unsafe.types.UTF8String /** @@ -30,7 +32,6 @@ object QuantitySupport { val VALUE_CANONICALIZED_FIELD_NAME = "_value_canonicalized" val CODE_CANONICALIZED_FIELD_NAME = "_code_canonicalized" - /** * Creates additional serializers for serialization of quantities. * @@ -40,26 +41,11 @@ object QuantitySupport { def createExtraSerializers(expression: Expression): Seq[(String, Expression)] = { val valueExp = Invoke(expression, "getValue", ObjectType(classOf[java.math.BigDecimal])) val codeExp = Invoke(expression, "getCode", ObjectType(classOf[java.lang.String])) - // TODO: Change to FlexDecimal - // val canonicalizedValue = StaticInvoke(classOf[Decimal], - // DecimalCustomCoder.decimalType, - // "apply", - // StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), - // "getCanonicalValue", Seq(valueExp, codeExp)) :: Nil) - -// val struct = CreateNamedStruct( -// allFields.flatMap({ case (name, serializer) => Seq(Literal(name), serializer) })) -// If(IsNull(expression), Literal.create(null, struct.dataType), struct) - - val canonicalizedValue = - StaticInvoke( - classOf[UTF8String], - DataTypes.StringType, - "fromString", - StaticInvoke(classOf[Ucum], ObjectType(classOf[java.lang.String]), - "getCanonicalValueAsString", Seq(valueExp, codeExp)) :: Nil) + createFlexiDecimalSerializer( + StaticInvoke(classOf[Ucum], ObjectType(classOf[java.math.BigDecimal]), + "getCanonicalValue", Seq(valueExp, codeExp))) val canonicalizedCode = StaticInvoke( @@ -81,7 +67,7 @@ object QuantitySupport { */ def createExtraSchemaFields(): Seq[StructField] = { Seq( - StructField(VALUE_CANONICALIZED_FIELD_NAME, FlexDecimal.DATA_TYPE), + StructField(VALUE_CANONICALIZED_FIELD_NAME, FlexiDecimal.DATA_TYPE), StructField(CODE_CANONICALIZED_FIELD_NAME, DataTypes.StringType) ) } diff --git a/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala new file mode 100644 index 0000000000..3ef8091a9f --- /dev/null +++ b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala @@ -0,0 +1,54 @@ +package au.csiro.pathling.sql.types + +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, IsNull, Literal} +import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} +import org.apache.spark.sql.functions.{lit, struct} +import org.apache.spark.sql.types.{DataTypes, Decimal, DecimalType, ObjectType} + +/** + * Helper class for serialization of FlexiDecimals. + */ +object FlexiDecimalSupport { + + def toLiteral(value: java.math.BigDecimal): Column = { + val normalizedValue = FlexiDecimal.normalize(value) + if (normalizedValue != null) { + struct( + lit(Decimal.apply(normalizedValue.unscaledValue())).cast(FlexiDecimal.DECIMAL_TYPE) + .as("value"), + lit(normalizedValue.scale()).as("scale") + ) + } else { + lit(null) + } + } + + def createFlexiDecimalSerializer(expression: Expression): Expression = { + // the expression is of type BigDecimal + // we want to encode it as a struct of two fields + // value -> the BigInteger representing digits + // scale -> the actual position of the decimal coma + + // TODO: Performance: consider if the normalized value could be cached in + // a variable + val normalizedExpression: Expression = StaticInvoke(classOf[FlexiDecimal], + ObjectType(classOf[java.math.BigDecimal]), + "normalize", + expression :: Nil) + + val valueExpression: Expression = StaticInvoke(classOf[Decimal], + FlexiDecimal.DECIMAL_TYPE, + "apply", + Invoke(normalizedExpression, "unscaledValue", + ObjectType(classOf[java.math.BigInteger])) :: Nil) + + val scaleExpression: Expression = Invoke(normalizedExpression, "scale", DataTypes.IntegerType) + + val struct = CreateNamedStruct(Seq( + Literal("value"), valueExpression, + Literal("scale"), scaleExpression + )) + If(IsNull(expression), Literal.create(null, struct.dataType), struct) + } +} diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java index 1d93fdb16b..ea465026a8 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/LightweightFhirEncodersTest.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import au.csiro.pathling.sql.types.FlexiDecimal; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import java.math.BigDecimal; @@ -100,13 +101,12 @@ public void assertSingletNestedExtension(final String expectedUrl, private static void assertQuantity(final Row quantityRow, final String canonicalizedValue, final String canonicalizedCode) { - // TODO: FlexDecimal - final String actualCanonicalizedValue = quantityRow.getString( - quantityRow.fieldIndex(QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME())); + final BigDecimal actualCanonicalizedValue = FlexiDecimal.fromValue(quantityRow.getStruct( + quantityRow.fieldIndex(QuantitySupport.VALUE_CANONICALIZED_FIELD_NAME()))); final String actualCanonicalizedCode = quantityRow.getString( quantityRow.fieldIndex(QuantitySupport.CODE_CANONICALIZED_FIELD_NAME())); - assertEquals(canonicalizedValue, actualCanonicalizedValue); + assertEquals(new BigDecimal(canonicalizedValue), actualCanonicalizedValue); assertEquals(canonicalizedCode, actualCanonicalizedCode); } diff --git a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java index fc530a32aa..35ee6e9270 100644 --- a/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java +++ b/encoders/src/test/java/au/csiro/pathling/encoders/SchemaConverterTest.java @@ -20,6 +20,7 @@ import au.csiro.pathling.encoders.datatypes.DataTypeMappings; import au.csiro.pathling.encoders.datatypes.R4DataTypeMappings; +import au.csiro.pathling.sql.types.FlexiDecimal; import ca.uhn.fhir.context.FhirContext; import java.util.Arrays; import java.util.List; @@ -96,7 +97,9 @@ private void traverseSchema(final DataType type, final Consumer cons if (type instanceof StructType) { final StructType structType = (StructType) type; consumer.accept(structType); - Arrays.stream(structType.fields()).forEach(f -> traverseSchema(f.dataType(), consumer)); + Arrays.stream(structType.fields()) + .filter(f -> !f.name().startsWith("_")) // filter out synthetic fields + .forEach(f -> traverseSchema(f.dataType(), consumer)); } else if (type instanceof ArrayType) { traverseSchema(((ArrayType) type).elementType(), consumer); } else if (type instanceof MapType) { @@ -417,8 +420,7 @@ private void assertQuantityType(final DataType quantityType) { assertTrue(getField(quantityType, true, "unit") instanceof StringType); assertTrue(getField(quantityType, true, "system") instanceof StringType); assertTrue(getField(quantityType, true, "code") instanceof StringType); - // TODO: FlexDecimal Change - assertTrue(getField(quantityType, true, "_value_canonicalized") instanceof StringType); + assertEquals(FlexiDecimal.DATA_TYPE, getField(quantityType, true, "_value_canonicalized")); assertTrue(getField(quantityType, true, "_code_canonicalized") instanceof StringType); } } diff --git a/encoders/src/test/java/au/csiro/pathling/sql/types/FlexiDecimalTest.java b/encoders/src/test/java/au/csiro/pathling/sql/types/FlexiDecimalTest.java new file mode 100644 index 0000000000..25179eb7f1 --- /dev/null +++ b/encoders/src/test/java/au/csiro/pathling/sql/types/FlexiDecimalTest.java @@ -0,0 +1,32 @@ +package au.csiro.pathling.sql.types; + +import org.junit.jupiter.api.Test; +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class FlexiDecimalTest { + + @Test + void testNormalize() { + // Normalize big decimal with negative scale + assertEquals(new BigDecimal("123000"), FlexiDecimal.normalize(new BigDecimal("1.23E+5"))); + assertEquals(new BigDecimal("99900000000000000000000000000000000000"), + FlexiDecimal.normalize(new BigDecimal("9.99E+37"))); + // To many significant digits --> null + assertNull(FlexiDecimal.normalize(new BigDecimal("1.0E+38"))); + + // normalize decimals to reduced precision + assertEquals(new BigDecimal("0.010000000000000000000000000000000000000"), + FlexiDecimal.normalize(new BigDecimal("0.010000000000000000000000000000000000000000000"))); + + assertEquals(FlexiDecimal.MAX_PRECISION, + FlexiDecimal.normalize(new BigDecimal("0.010000000000000000000000000000000000000000000")) + .precision()); + + assertEquals(new BigDecimal("0.015555555555555555555555555555555555556"), + FlexiDecimal.normalize(new BigDecimal( + "0.01555555555555555555555555555555555555555555555555555555555555555555555555"))); + } +} diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java index d68222e4ac..6eb9bc19b8 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java @@ -10,7 +10,7 @@ import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; -import au.csiro.pathling.sql.types.FlexDecimal; +import au.csiro.pathling.sql.types.FlexiDecimal; import org.apache.spark.sql.Column; import javax.annotation.Nonnull; import java.util.function.BiFunction; @@ -50,7 +50,7 @@ private static BiFunction wrap( @Override public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(FlexDecimal::equals).apply(left, right); + return wrap(FlexiDecimal::equals).apply(left, right); } // @Override @@ -60,22 +60,22 @@ public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) @Override public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(FlexDecimal::lt).apply(left, right); + return wrap(FlexiDecimal::lt).apply(left, right); } @Override public Column lessThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(FlexDecimal::lte).apply(left, right); + return wrap(FlexiDecimal::lte).apply(left, right); } @Override public Column greaterThan(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(FlexDecimal::gt).apply(left, right); + return wrap(FlexiDecimal::gt).apply(left, right); } @Override public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Column right) { - return wrap(FlexDecimal::gte).apply(left, right); + return wrap(FlexiDecimal::gte).apply(left, right); } /** diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index daad92e05c..4737952d59 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -18,7 +18,7 @@ import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.fhirpath.literal.NullLiteralPath; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; -import au.csiro.pathling.sql.types.FlexDecimal; +import au.csiro.pathling.sql.types.FlexiDecimal; import com.google.common.collect.ImmutableSet; import java.util.Optional; import java.util.function.BiFunction; @@ -83,13 +83,13 @@ private static BiFunction getMathOperation( @Nonnull final MathOperation operation) { switch (operation) { case ADDITION: - return FlexDecimal::plus; + return FlexiDecimal::plus; case MULTIPLICATION: - return FlexDecimal::multiply; + return FlexiDecimal::multiply; case DIVISION: - return FlexDecimal::divide; + return FlexiDecimal::divide; case SUBTRACTION: - return FlexDecimal::minus; + return FlexiDecimal::minus; default: throw new AssertionError("Unsupported math operation encountered: " + operation); } @@ -162,7 +162,7 @@ public static Function buildMathOperation(@Nonnull fina final Column resultStruct = QuantityEncoding.toStruct( sourceContext.getField("id"), - resultColumn, + FlexiDecimal.to_decimal(resultColumn), // NOTE: This (setting value_scale to null) works because we never decode this struct to a Quantity. // The only Quantities that are decoded are calendar duration quantities parsed from literals. lit(null), diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 87c02db035..f76355b368 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -19,6 +19,8 @@ import javax.annotation.Nullable; import au.csiro.pathling.encoders.terminology.ucum.Ucum; import au.csiro.pathling.fhirpath.CalendarDurationUtils; +import au.csiro.pathling.sql.types.FlexiDecimal; +import au.csiro.pathling.sql.types.FlexiDecimalSupport; import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; @@ -28,7 +30,6 @@ import org.apache.spark.sql.types.MetadataBuilder; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; -import org.hl7.fhir.r4.model.Coding; import org.hl7.fhir.r4.model.Quantity; import org.hl7.fhir.r4.model.Quantity.QuantityComparator; @@ -68,10 +69,10 @@ public static Row encode(@Nullable final Quantity quantity, boolean includeScale } final BigDecimal value = quantity.getValue(); @Nullable final String code = quantity.getCode(); - final String canonicalizedValue; + final BigDecimal canonicalizedValue; final String canonicalizedCode; if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { - canonicalizedValue = Ucum.getCanonicalValueAsString(value, code); + canonicalizedValue = Ucum.getCanonicalValue(value, code); canonicalizedCode = Ucum.getCanonicalCode(value, code); } else { canonicalizedValue = null; @@ -79,7 +80,7 @@ public static Row encode(@Nullable final Quantity quantity, boolean includeScale } final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); - // TODO: The null scale support it a temporary measure becasue we currently + // TODO: The null scale support it a temporary measure because we currently // cannot encode the scale of the results of arithmetic operations. return RowFactory.create(quantity.getId(), quantity.getValue(), @@ -87,7 +88,8 @@ public static Row encode(@Nullable final Quantity quantity, boolean includeScale ? quantity.getValue().scale() : null, comparator, - quantity.getUnit(), quantity.getSystem(), quantity.getCode(), canonicalizedValue, + quantity.getUnit(), quantity.getSystem(), quantity.getCode(), + FlexiDecimal.toValue(canonicalizedValue), canonicalizedCode, null /* _fid */); } @@ -153,9 +155,8 @@ public static StructType dataType() { final StructField unit = new StructField("unit", DataTypes.StringType, true, metadata); final StructField system = new StructField("system", DataTypes.StringType, true, metadata); final StructField code = new StructField("code", DataTypes.StringType, true, metadata); - // TODO: FlexDecimal final StructField canonicalizedValue = new StructField(CANONICALIZED_VALUE_COLUMN, - DataTypes.StringType, true, metadata); + FlexiDecimal.DATA_TYPE, true, metadata); final StructField canonicalizedCode = new StructField(CANONICALIZED_CODE_COLUMN, DataTypes.StringType, true, metadata); final StructField fid = new StructField("_fid", DataTypes.IntegerType, true, @@ -219,18 +220,18 @@ public static Column encodeLiteral(@Nonnull final Quantity quantity) { final BigDecimal value = quantity.getValue(); // FlexDecima; - final String canonicalizedValue; + final BigDecimal canonicalizedValue; final String canonicalizedCode; if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { // If it is a UCUM Quantity, use the UCUM library to canonicalize the value and code. - canonicalizedValue = Ucum.getCanonicalValueAsString(value, quantity.getCode()); + canonicalizedValue = Ucum.getCanonicalValue(value, quantity.getCode()); canonicalizedCode = Ucum.getCanonicalCode(value, quantity.getCode()); } else if (CalendarDurationUtils.isCalendarDuration(quantity) && CALENDAR_DURATION_TO_UCUM.containsKey(quantity.getCode())) { // If it is a (supported) calendar duration, get the corresponding UCUM unit and then use the // UCUM library to canonicalize the value and code. final String resolvedCode = CALENDAR_DURATION_TO_UCUM.get(quantity.getCode()); - canonicalizedValue = Ucum.getCanonicalValueAsString(value, resolvedCode); + canonicalizedValue = Ucum.getCanonicalValue(value, resolvedCode); canonicalizedCode = Ucum.getCanonicalCode(value, resolvedCode); } else { // If it is neither a UCUM Quantity nor a calendar duration, it will not have a canonicalized @@ -247,7 +248,7 @@ public static Column encodeLiteral(@Nonnull final Quantity quantity) { lit(quantity.getUnit()), lit(quantity.getSystem()), lit(quantity.getCode()), - lit(canonicalizedValue), + FlexiDecimalSupport.toLiteral(canonicalizedValue), lit(canonicalizedCode), lit(null)); } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java index 80f1600176..a27630ac5b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/EqualityOperatorQuantityTest.java @@ -130,7 +130,7 @@ void setUp() { .withRow("patient-8", rowFromQuantity(quantity7)) // 60 s .withRow("patient-9", rowFromQuantity(quantity8)) // 1000 ms .withRow("patient-a", rowForUcumQuantity("1000", "mmol")) // 1000 mmol - .withRow("patient-b", rowForUcumQuantity("49", "%")) // 50 % + .withRow("patient-b", rowForUcumQuantity("49", "%")) // 49 % .withRow("patient-c", rowFromQuantity(nonUcumQuantity)) // non-ucum .buildWithStructValue(); left = new ElementPathBuilder(spark) @@ -183,7 +183,7 @@ void equals() { final OperatorInput input = new OperatorInput(parserContext, left, right); final Operator equalityOperator = Operator.getInstance("="); final FhirPath result = equalityOperator.invoke(input); - + assertThat(result).selectOrderedResult().hasRows( RowFactory.create("patient-1", true), // 500 mg = 500 mg RowFactory.create("patient-2", true), // 500 mg = 0.5 g @@ -257,8 +257,8 @@ void ucumLiteralEquals() { RowFactory.create("patient-8", null), // 60 s = 0.5 mg RowFactory.create("patient-9", null), // 1000 ms = 0.5 mg RowFactory.create("patient-a", null), // 1000 mmol = 0.5 mg - RowFactory.create("patient-b", null), // 49 % != = 0.5 mg - RowFactory.create("patient-c", null) // non-ucum != 0.5 mg + RowFactory.create("patient-b", null), // 49 % = 0.5 mg + RowFactory.create("patient-c", null) // non-ucum = 0.5 mg ); input = new OperatorInput(parserContext, left, ucumQuantityLiteral3); @@ -275,26 +275,26 @@ void ucumLiteralEquals() { RowFactory.create("patient-8", null), // 60 s = 1.8 m RowFactory.create("patient-9", null), // 1000 ms = 1.8 m RowFactory.create("patient-a", null), // 1000 mmol = 1.8 m - RowFactory.create("patient-b", null), // 49 % != = 1.8 m - RowFactory.create("patient-c", null) // non-ucum != 1.8 m + RowFactory.create("patient-b", null), // 49 % = 1.8 m + RowFactory.create("patient-c", null) // non-ucum = 1.8 m ); input = new OperatorInput(parserContext, left, ucumQuantityLiteralNoUnit); result = equalityOperator.invoke(input); assertThat(result).selectOrderedResult().hasRows( - RowFactory.create("patient-1", null), // 500 mg = 1.8 m - RowFactory.create("patient-2", null), // 500 mg = 1.8 m - RowFactory.create("patient-3", null), // 500 mg = 1.8 m - RowFactory.create("patient-4", null), // 650 mg = 1.8 m - RowFactory.create("patient-5", null), // {} = 1.8 m - RowFactory.create("patient-6", null), // 500 mg = 1.8 m - RowFactory.create("patient-7", null), // 30 d = 1.8 m - RowFactory.create("patient-8", null), // 60 s = 1.8 m - RowFactory.create("patient-9", null), // 1000 ms = 1.8 m - RowFactory.create("patient-a", false), // 1000 mmol = 1.8 m - RowFactory.create("patient-b", true), // 49 % != = 1.8 m - RowFactory.create("patient-c", null) // non-ucum != 1.8 m + RowFactory.create("patient-1", null), // 500 mg = 0.49 '1' + RowFactory.create("patient-2", null), // 500 mg = 0.49 '1' + RowFactory.create("patient-3", null), // 500 mg = 0.49 '1' + RowFactory.create("patient-4", null), // 650 mg = 0.49 '1' + RowFactory.create("patient-5", null), // {} = 0.49 '1' + RowFactory.create("patient-6", null), // 500 mg = 0.49 '1 + RowFactory.create("patient-7", null), // 30 d = 0.49 '1' + RowFactory.create("patient-8", null), // 60 s = 0.49 '1' + RowFactory.create("patient-9", null), // 1000 ms = 0.49 '1' + RowFactory.create("patient-a", false), // 1000 mmol = 0.49 '1' + RowFactory.create("patient-b", true), // 49 % = 0.49 '1' + RowFactory.create("patient-c", null) // non-ucum = 0.49 '1' ); input = new OperatorInput(parserContext, ucumQuantityLiteral1, ucumQuantityLiteral1); @@ -369,7 +369,7 @@ void ucumLiteralNotEquals() { input = new OperatorInput(parserContext, left, ucumQuantityLiteralNoUnit); result = equalityOperator.invoke(input); - + assertThat(result).selectOrderedResult().hasRows( RowFactory.create("patient-1", null), // 500 mg != 0.49 '' RowFactory.create("patient-2", null), // 500 mg != 0.49 '' diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java index 0d7c01b5e4..5465b3bcb1 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/QuantityOperatorsPrecisionTest.java @@ -15,6 +15,7 @@ import au.csiro.pathling.test.builders.ParserContextBuilder; import au.csiro.pathling.test.helpers.TestHelpers; import ca.uhn.fhir.context.FhirContext; +import com.google.common.collect.ImmutableSet; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -34,6 +35,7 @@ import java.math.BigDecimal; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static au.csiro.pathling.test.assertions.Assertions.assertThat; @@ -70,6 +72,14 @@ public class QuantityOperatorsPrecisionTest { static final String FULL_DECIMAL_02 = createSpanningDecimal(9, 26, 2, 6).toString(); // 9e26 + 0.00002 + // These units (prefixes) results in overflow for largest supported decimals (e.g. 9e26 'Tm') + static final Set UNSUPPORTED_FULL_DECIMAL_UNITS = ImmutableSet.of("Ym", "Zm", "Em", "Pm", + "Tm"); + + // These mol prefixes results in overflow for reasonable decimals (e.g. 9e3 'Tmol') + static final Set UNSUPPORTED_RESONABLE_DECIMAL_MOL_UNITS = ImmutableSet.of("Ymol", "Zmol", + "Emol", "Pmol", "Tmol"); + @BeforeEach void setUp() { } @@ -115,11 +125,20 @@ private static BigDecimal createSpanningDecimal(int leftValue, int leftScale, in @Nonnull private static List createResult(@Nonnull final List unitRange, boolean result) { + return createResult(unitRange, result, Collections.emptySet()); + } + + @Nonnull + private static List createResult(@Nonnull final List unitRange, boolean result, + @Nonnull final Set outOfRangeUnits) { return unitRange.stream().map( unit -> - RowFactory.create(unitToRowId(unit), result)).collect(Collectors.toList()); + RowFactory.create(unitToRowId(unit), outOfRangeUnits.contains(unit) + ? null + : result)).collect(Collectors.toList()); } + @Nonnull private FhirPath callOperator(@Nonnull final ElementPath left, @Nonnull final String operator, @Nonnull final ElementPath right) { @@ -168,18 +187,18 @@ void equalityPrecisionForFullDecimals() { final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); final FhirPath result = callOperator(left, "=", right); - assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + assertThat(result).selectResult() + .hasRows(createResult(unitRange, true, UNSUPPORTED_FULL_DECIMAL_UNITS)); } - + @Test void nonEqualityPrecisionForFullDecimals() { final List unitRange = getAllPrefixedUnits("m"); final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); - left.getDataset().collectAsList().forEach(System.out::println); final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_02, unitRange); - right.getDataset().collectAsList().forEach(System.out::println); final FhirPath result = callOperator(left, "!=", right); - assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + assertThat(result).selectResult() + .hasRows(createResult(unitRange, true, UNSUPPORTED_FULL_DECIMAL_UNITS)); } @Test @@ -188,7 +207,8 @@ void comparisonPrecisionForFullDecimals() { final ElementPath left = buildQuantityPathForUnits(FULL_DECIMAL_01, unitRange); final ElementPath right = buildQuantityPathForUnits(FULL_DECIMAL_02, unitRange); final FhirPath result = callOperator(left, "<", right); - assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + assertThat(result).selectResult() + .hasRows(createResult(unitRange, true, UNSUPPORTED_FULL_DECIMAL_UNITS)); } @Test @@ -197,7 +217,8 @@ void equalityPrecisionForReasonableDecimalsWithMols() { final ElementPath left = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); final ElementPath right = buildQuantityPathForUnits(REASONABLE_DECIMAL_01, unitRange); final FhirPath result = callOperator(left, "=", right); - assertThat(result).selectResult().hasRows(createResult(unitRange, true)); + assertThat(result).selectResult() + .hasRows(createResult(unitRange, true, UNSUPPORTED_RESONABLE_DECIMAL_MOL_UNITS)); } } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/AbstractParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/AbstractParserTest.java index 8ebed68c26..df7ed58a07 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/AbstractParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/AbstractParserTest.java @@ -9,9 +9,11 @@ import static au.csiro.pathling.test.assertions.Assertions.assertThat; import static org.mockito.Mockito.when; +import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.fhirpath.ResourcePath; import au.csiro.pathling.io.Database; +import au.csiro.pathling.terminology.TerminologyService; import au.csiro.pathling.test.SharedMocks; import au.csiro.pathling.test.TimingExtension; import au.csiro.pathling.test.assertions.FhirPathAssertion; @@ -42,6 +44,12 @@ public class AbstractParserTest { @Autowired FhirContext fhirContext; + @Autowired + TerminologyService terminologyService; + + @Autowired + FhirEncoders fhirEncoders; + @Autowired TerminologyServiceFactory terminologyServiceFactory; @@ -93,4 +101,9 @@ protected FhirPathAssertion assertThatResultOf(@Nonnull final ResourceType resou return assertThat(resourceParser.parse(expression)); } + @SuppressWarnings("SameParameterValue") + FhirPathAssertion assertThatResultOf(final String expression) { + return assertThat(parser.parse(expression)); + } + } diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java index 3cae75323e..60b8c8257e 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/DateTimeArithmeticParserTest.java @@ -15,7 +15,7 @@ import org.hl7.fhir.r4.model.Enumerations.ResourceType; import org.junit.jupiter.api.Test; -public class DateTimeArithmeticParserTest extends ParserTest { +public class DateTimeArithmeticParserTest extends AbstractParserTest { @Test void lengthOfEncounter() { diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java index a27d1a9a40..d684e3898f 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/ParserTest.java @@ -61,17 +61,7 @@ * @author Piotr Szul */ public class ParserTest extends AbstractParserTest { - - @Autowired - TerminologyService terminologyService; - - @Autowired - FhirEncoders fhirEncoders; - - FhirPathAssertion assertThatResultOf(final String expression) { - return assertThat(parser.parse(expression)); - } - + @SuppressWarnings("SameParameterValue") private T assertThrows(final Class errorType, final String expression) { return Assertions.assertThrows(errorType, () -> parser.parse(expression)); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java index 0230f3dd3a..6e94165c4d 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/parser/QuantityParserTest.java @@ -13,7 +13,7 @@ import org.hl7.fhir.r4.model.Enumerations.ResourceType; import org.junit.jupiter.api.Test; -public class QuantityParserTest extends ParserTest { +public class QuantityParserTest extends AbstractParserTest { @Test void lengthObservationComparison() { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java index 3cefd4196c..1a5cfd9666 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -4,7 +4,6 @@ import au.csiro.pathling.jmh.AbstractJmhSpringBootState; import au.csiro.pathling.sql.types.FlexDecimal; import au.csiro.pathling.sql.types.FlexiDecimal; -import au.csiro.pathling.sql.types.UcumDecimal; import au.csiro.pathling.test.builders.DatasetBuilder; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; @@ -31,7 +30,8 @@ public class DecimalBenchmark { private static final int ROWS = 1000000; - private static final BigDecimal LEFT_DECIMAL = new BigDecimal("12345678901234567890123456.123456"); + private static final BigDecimal LEFT_DECIMAL = new BigDecimal( + "12345678901234567890123456.123456"); private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("0.12345678901234567890123456"); @State(Scope.Benchmark) @@ -53,9 +53,7 @@ public void setUp() { .withColumn("leftFlexDecimal", FlexDecimal.DATA_TYPE) .withColumn("rightFlexDecimal", FlexDecimal.DATA_TYPE) .withColumn("leftFlexiDecimal", FlexiDecimal.DATA_TYPE) - .withColumn("rightFlexiDecimal", FlexiDecimal.DATA_TYPE) - .withColumn("leftUcumDecimal", UcumDecimal.DATA_TYPE) - .withColumn("rightUcumDecimal", UcumDecimal.DATA_TYPE); + .withColumn("rightFlexiDecimal", FlexiDecimal.DATA_TYPE); for (int i = 0; i < ROWS; i++) { datasetBuilder = datasetBuilder.withRow( @@ -64,9 +62,7 @@ public void setUp() { FlexDecimal.toValue(LEFT_DECIMAL), FlexDecimal.toValue(RIGHT_DECIMAL), FlexiDecimal.toValue(LEFT_DECIMAL), - FlexiDecimal.toValue(RIGHT_DECIMAL), - LEFT_DECIMAL.toString(), - RIGHT_DECIMAL.toString() + FlexiDecimal.toValue(RIGHT_DECIMAL) ); } dataset = datasetBuilder.build().cache(); @@ -135,6 +131,7 @@ public void lt_flexDec_Benchmark(final Blackhole bh, bh.consume(ds.collectQuery( FlexDecimal.lt(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); } + @Benchmark public void multiply_flexiDec_Benchmark(final Blackhole bh, final DatasetState ds) { @@ -163,31 +160,4 @@ public void lt_flexiDec_Benchmark(final Blackhole bh, FlexiDecimal.lt(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); } - @Benchmark - public void multiply_ucumDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - UcumDecimal.multiply(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); - } - - @Benchmark - public void add_ucumDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - UcumDecimal.plus(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); - } - - @Benchmark - public void equals_ucumDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - UcumDecimal.equals(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); - } - - @Benchmark - public void lt_ucumDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - UcumDecimal.lt(ds.col("leftUcumDecimal"), ds.col("rightUcumDecimal")))); - } } From 1a8c9352c07ff5b4f25ae98ae19c6bcbf28158c4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Fri, 14 Oct 2022 20:34:22 +1000 Subject: [PATCH 64/83] Enable arbitrary configuration of Hadoop AWS through config variables --- .../pathling/config/StorageConfiguration.java | 56 ------------------ .../java/au/csiro/pathling/spark/Spark.java | 59 ++++++------------- .../src/main/resources/application.yml | 39 ++++++++---- 3 files changed, 45 insertions(+), 109 deletions(-) diff --git a/fhir-server/src/main/java/au/csiro/pathling/config/StorageConfiguration.java b/fhir-server/src/main/java/au/csiro/pathling/config/StorageConfiguration.java index f9535f347f..cfbe3ec7e3 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/config/StorageConfiguration.java +++ b/fhir-server/src/main/java/au/csiro/pathling/config/StorageConfiguration.java @@ -6,15 +6,10 @@ package au.csiro.pathling.config; -import java.util.Optional; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; import javax.validation.constraints.Pattern; import javax.validation.constraints.Size; import lombok.Data; -import lombok.ToString; /** * Configuration relating to the storage of data. @@ -38,55 +33,4 @@ public class StorageConfiguration { @Size(min = 1, max = 50) private String databaseName; - @NotNull - private Aws aws; - - /** - * Configuration relating to storage of data using Amazon Web Services (AWS). - */ - @Data - public static class Aws { - - /** - * Public buckets can be accessed by default, set this to false to access protected buckets. - */ - @NotNull - private boolean anonymousAccess; - - /** - * Authentication details for connecting to a protected Amazon S3 bucket. - */ - @Nullable - private String accessKeyId; - - /** - * Authentication details for connecting to a protected Amazon S3 bucket. - */ - @Nullable - @ToString.Exclude - private String secretAccessKey; - - /** - * The ARN of an IAM role that should be assumed using STS. - */ - @Nullable - private String assumedRole; - - @Nonnull - public Optional getAccessKeyId() { - return Optional.ofNullable(accessKeyId); - } - - @Nonnull - public Optional getSecretAccessKey() { - return Optional.ofNullable(secretAccessKey); - } - - @Nonnull - public Optional getAssumedRole() { - return Optional.ofNullable(assumedRole); - } - - } - } diff --git a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java index 8e092e1905..ec4a600844 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java +++ b/fhir-server/src/main/java/au/csiro/pathling/spark/Spark.java @@ -6,14 +6,15 @@ package au.csiro.pathling.spark; -import au.csiro.pathling.config.Configuration; -import au.csiro.pathling.config.StorageConfiguration.Aws; import au.csiro.pathling.async.SparkListener; +import au.csiro.pathling.config.Configuration; import au.csiro.pathling.sql.CodingToLiteral; import au.csiro.pathling.sql.PathlingStrategy; import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.SparkSession; @@ -52,7 +53,11 @@ public static SparkSession build(@Nonnull final Configuration configuration, @Nonnull final Environment environment, @Nonnull final Optional sparkListener) { log.debug("Creating Spark session"); - resolveSparkConfiguration(environment); + + // Pass through Spark configuration. + resolveThirdPartyConfiguration(environment, List.of("spark."), + property -> System.setProperty(property, + Objects.requireNonNull(environment.getProperty(property)))); final SparkSession spark = SparkSession.builder() .appName(configuration.getSpark().getAppName()) @@ -64,54 +69,26 @@ public static SparkSession build(@Nonnull final Configuration configuration, spark.udf() .register(CodingToLiteral.FUNCTION_NAME, new CodingToLiteral(), DataTypes.StringType); - // Configure AWS driver and credentials. - configureAwsDriver(configuration, spark); + // Pass through Hadoop AWS configuration. + resolveThirdPartyConfiguration(environment, List.of("fs.s3a."), + property -> spark.sparkContext().hadoopConfiguration().set(property, + Objects.requireNonNull(environment.getProperty(property)))); return spark; } - private static void configureAwsDriver(@Nonnull final Configuration configuration, - @Nonnull final SparkSession spark) { - final Aws awsConfig = configuration.getStorage().getAws(); - final org.apache.hadoop.conf.Configuration hadoopConfig = spark.sparkContext() - .hadoopConfiguration(); - - // We need to use the anonymous credentials provider if we are not using AWS credentials. - if (awsConfig.isAnonymousAccess()) { - hadoopConfig.set("fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"); - } - - // Set credentials if provided. - awsConfig.getAccessKeyId() - .ifPresent(accessKeyId -> hadoopConfig.set("fs.s3a.access.key", accessKeyId)); - awsConfig.getSecretAccessKey() - .ifPresent(secretAccessKey -> hadoopConfig.set("fs.s3a.secret.key", secretAccessKey)); - hadoopConfig.set("fs.s3a.connection.maximum", "100"); - hadoopConfig.set("fs.s3a.committer.magic.enabled", "true"); - hadoopConfig.set("fs.s3a.committer.name", "magic"); - - // Assume role if configured. - awsConfig.getAssumedRole() - .ifPresent(assumedRole -> { - hadoopConfig.set("fs.s3a.aws.credentials.provider", - "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); - hadoopConfig.set("fs.s3a.assumed.role.arn", assumedRole); - }); - } - - private static void resolveSparkConfiguration(@Nonnull final PropertyResolver resolver) { - // This goes through the properties within the Spring configuration and copies the Spark - // configuration into Java system properties, which Spark will then pick up. + private static void resolveThirdPartyConfiguration(@Nonnull final PropertyResolver resolver, + @Nonnull final List prefixes, @Nonnull final Consumer setter) { + // This goes through the properties within the Spring configuration and invokes the provided + // setter function for each property that matches one of the supplied prefixes. final MutablePropertySources propertySources = ((AbstractEnvironment) resolver) .getPropertySources(); propertySources.stream() .filter(propertySource -> propertySource instanceof EnumerablePropertySource) .flatMap(propertySource -> Arrays .stream(((EnumerablePropertySource) propertySource).getPropertyNames())) - .filter(property -> property.startsWith("spark.")) - .forEach(property -> System.setProperty(property, - Objects.requireNonNull(resolver.getProperty(property)))); + .filter(property -> prefixes.stream().anyMatch(property::startsWith)) + .forEach(setter); } } diff --git a/fhir-server/src/main/resources/application.yml b/fhir-server/src/main/resources/application.yml index 3b8dc796fa..28f3b73d80 100644 --- a/fhir-server/src/main/resources/application.yml +++ b/fhir-server/src/main/resources/application.yml @@ -42,18 +42,6 @@ pathling: # The subdirectory within the warehouse path used to read and write data. databaseName: default - # Configuration relating to accessing data hosted within Amazon Web Services. - aws: - # Public S3 buckets can be accessed by default, set this to false to access protected buckets. - anonymousAccess: true - - # Authentication details for connecting to protected Amazon S3 locations. - # accessKeyId: [AWS access key ID] - # secretAccessKey: [AWS secret access key] - - # The ARN of an IAM role to be assumed using STS. - # assumedRole: [ARN of IAM role] - terminology: # Enables the use of terminology functions. enabled: true @@ -200,3 +188,30 @@ spark: enabled: true scheduler: mode: FAIR + +# Use this section to set or override configuration relating to S3 configuration. +# See: https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html +fs: + s3a: + aws: + # credentials: + # provider: org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider + # provider: org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider + # + # For use with: SimpleAWSCredentialsProvider + # access: + # key: [access key] + # secret: + # key: [secret key] + # + # For use with: AssumedRoleCredentialProvider + # assumed: + # role: + # arn: [role ARN] + # + connection: + maximum: 100 + committer: + name: magic + magic: + enabled: true From 9f4ee25a21735fbda9b49f54f9f6bcbea968cd96 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Sat, 15 Oct 2022 08:47:48 +1000 Subject: [PATCH 65/83] Update version to 6.0.0-SNAPSHOT and create pre-release workflow --- .github/workflows/pre-release.yml | 45 ++++++++++++++++++++ encoders/pom.xml | 2 +- fhir-server/azure-pipelines.yml | 68 ------------------------------- fhir-server/pom.xml | 43 ++++++++++++++++++- lib/import/pom.xml | 2 +- lib/js/pom.xml | 2 +- lib/python/pom.xml | 2 +- library-api/pom.xml | 2 +- pom.xml | 2 +- site/pom.xml | 2 +- terminology/pom.xml | 2 +- utilities/pom.xml | 2 +- 12 files changed, 96 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/pre-release.yml delete mode 100644 fhir-server/azure-pipelines.yml diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml new file mode 100644 index 0000000000..fa686deee0 --- /dev/null +++ b/.github/workflows/pre-release.yml @@ -0,0 +1,45 @@ +# This workflow creates a pre-release Docker image. + +name: Deploy pre-release Docker image + +on: + push: + branches: + - 'release/*' + +jobs: + deploy-docker: + name: Docker Hub + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + # This is required so that git-commit-id-plugin can find the latest tag. + fetch-depth: 0 + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 11 + distribution: 'zulu' + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Run the deploy goal with Maven + run: | + TAG=$(echo '${{ github.ref }}' | sed 's/refs\/heads\/release\///') + mvn --batch-mode deploy \ + -pl fhir-server -am \ + -PdockerPreRelease \ + -Dpathling.fhirServerDockerTag=$TAG \ + -DskipTests -DskipScalaDocs + timeout-minutes: 30 diff --git a/encoders/pom.xml b/encoders/pom.xml index 7cec9fef30..80a929b0aa 100644 --- a/encoders/pom.xml +++ b/encoders/pom.xml @@ -20,7 +20,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT encoders jar diff --git a/fhir-server/azure-pipelines.yml b/fhir-server/azure-pipelines.yml deleted file mode 100644 index 9a04ce812e..0000000000 --- a/fhir-server/azure-pipelines.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: pathling-$(Date:yyyyMMdd)$(Rev:.r) - -trigger: none - -pr: - branches: - exclude: - - dependabot/* - -pool: - vmImage: 'ubuntu-latest' - -variables: - mavenCache: $(Pipeline.Workspace)/.m2/repository - mavenOptions: '-Dmaven.repo.local=$(mavenCache)' - branchTag: $[replace(variables['System.PullRequest.SourceBranch'], '/', '-')] - -stages: - - stage: build - displayName: Build - jobs: - - job: build - displayName: Build - steps: - - task: Cache@2 - displayName: Cache Maven local repo - inputs: - key: 'maven | "$(Agent.OS)" | **/pom.xml' - restoreKeys: | - maven | "$(Agent.OS)" - maven - path: $(mavenCache) - - task: Maven@3 - displayName: Build and verify - inputs: - options: '-pl fhir-server -am -Pdocker -Dpathling.systemTest.auth.clientSecret=$(testClientSecret)' - mavenOptions: '$(mavenOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: '1.11' - jdkArchitectureOption: 'x64' - goals: 'verify' - timeoutInMinutes: 60 - - task: Docker@2 - condition: succeeded() - displayName: Tag image - inputs: - containerRegistry: $(containerRegistry) - repository: $(dockerRepo) - command: tag - arguments: aehrc/pathling:latest $(dockerRegistry)/$(dockerRepo):$(Build.BuildNumber) - - task: Docker@2 - condition: succeeded() - displayName: Tag image - inputs: - containerRegistry: $(containerRegistry) - repository: $(dockerRepo) - command: tag - arguments: aehrc/pathling:latest $(dockerRegistry)/$(dockerRepo):$(branchTag) - - task: Docker@2 - condition: succeeded() - displayName: Push image - inputs: - containerRegistry: $(containerRegistry) - repository: $(dockerRepo) - command: push - tags: | - $(Build.BuildNumber) - $(branchTag) diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index fd6d6d2ab1..b6cc679234 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT fhir-server jar @@ -582,6 +582,47 @@ + + dockerPreRelease + + + + com.google.cloud.tools + jib-maven-plugin + + + deploy + deploy + + build + + + + ${pathling.dockerBaseImage} + + + amd64 + linux + + + arm64 + linux + + + + + ${pathling.fhirServerDockerRepo} + + ${pathling.fhirServerDockerTag}-develop + + + + + + + + + testTranche1 diff --git a/lib/import/pom.xml b/lib/import/pom.xml index 5dacc5bcec..2c8bec50c4 100644 --- a/lib/import/pom.xml +++ b/lib/import/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT ../../pom.xml import diff --git a/lib/js/pom.xml b/lib/js/pom.xml index a2ff07adfe..4e9791d5b3 100644 --- a/lib/js/pom.xml +++ b/lib/js/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT ../../pom.xml js diff --git a/lib/python/pom.xml b/lib/python/pom.xml index 6d6b656dbe..5148fc8619 100644 --- a/lib/python/pom.xml +++ b/lib/python/pom.xml @@ -7,7 +7,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT ../../pom.xml python diff --git a/library-api/pom.xml b/library-api/pom.xml index c5ff1384d9..1cc2049e1a 100644 --- a/library-api/pom.xml +++ b/library-api/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.4.0 + 6.0.0-SNAPSHOT library-api jar diff --git a/pom.xml b/pom.xml index 484b881230..08f4a45b3b 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT pom Pathling diff --git a/site/pom.xml b/site/pom.xml index f8549a1cd1..2e4d66665e 100644 --- a/site/pom.xml +++ b/site/pom.xml @@ -13,7 +13,7 @@ au.csiro.pathling pathling - 5.4.0 + 6.0.0-SNAPSHOT ../pom.xml site diff --git a/terminology/pom.xml b/terminology/pom.xml index 86a87381bb..cd3c288f1b 100644 --- a/terminology/pom.xml +++ b/terminology/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.4.0 + 6.0.0-SNAPSHOT terminology jar diff --git a/utilities/pom.xml b/utilities/pom.xml index 8633ca1adf..6a8d438dfe 100644 --- a/utilities/pom.xml +++ b/utilities/pom.xml @@ -8,7 +8,7 @@ pathling au.csiro.pathling - 5.4.0 + 6.0.0-SNAPSHOT utilities jar From 91570b4bcb7ce9cd4df84bb21196bcad657921a4 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 18 Oct 2022 09:55:31 +1000 Subject: [PATCH 66/83] Don't set latest tag upon deployment of pre-release image --- fhir-server/pom.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index b6cc679234..64a629a3df 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -611,10 +611,7 @@ - ${pathling.fhirServerDockerRepo} - - ${pathling.fhirServerDockerTag}-develop - + ${pathling.fhirServerDockerRepo}:${pathling.fhirServerDockerTag}-develop From 6877729328953d0b4b3144297af6b232e985f445 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 11:15:55 +1000 Subject: [PATCH 67/83] Add standard JVM args to benchmark execution --- fhir-server/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 72e3710956..51c24ea59d 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -429,6 +429,9 @@ + -Xmx4g + -ea + -Duser.timezone=UTC ${project.build.directory}/benchmark From 2b9edfab680e90b449654060cad939d02d791571 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 11:59:45 +1000 Subject: [PATCH 68/83] Fix failing tests --- .../integration/OperationDefinitionTest.java | 2 +- ...pabilityStatement.CapabilityStatement.json | 848 +++++++++--------- pom.xml | 2 +- 3 files changed, 426 insertions(+), 426 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/integration/OperationDefinitionTest.java b/fhir-server/src/test/java/au/csiro/pathling/test/integration/OperationDefinitionTest.java index 280ea44a69..42df13b8cd 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/integration/OperationDefinitionTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/integration/OperationDefinitionTest.java @@ -38,7 +38,7 @@ class OperationDefinitionTest extends IntegrationTest { private static final List OPERATIONS = List.of("aggregate", "search", "extract", "import", "result", "job"); - private static final String SUFFIX = "5"; + private static final String SUFFIX = "6"; @Test void operationDefinitions() throws MalformedURLException, URISyntaxException { diff --git a/fhir-server/src/test/resources/responses/CapabilityStatementTest/capabilityStatement.CapabilityStatement.json b/fhir-server/src/test/resources/responses/CapabilityStatementTest/capabilityStatement.CapabilityStatement.json index 6a8cb2dda9..1880c7b2d1 100644 --- a/fhir-server/src/test/resources/responses/CapabilityStatementTest/capabilityStatement.CapabilityStatement.json +++ b/fhir-server/src/test/resources/responses/CapabilityStatementTest/capabilityStatement.CapabilityStatement.json @@ -1,6 +1,6 @@ { "resourceType": "CapabilityStatement", - "url": "https://pathling.csiro.au/fhir/CapabilityStatement/pathling-fhir-api-5", + "url": "https://pathling.csiro.au/fhir/CapabilityStatement/pathling-fhir-api-6", "name": "pathling-fhir-api", "title": "Pathling FHIR API", "status": "active", @@ -40,15 +40,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -69,15 +69,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -98,15 +98,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -127,15 +127,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -156,15 +156,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -185,15 +185,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -214,15 +214,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -243,15 +243,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -272,15 +272,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -301,15 +301,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -330,15 +330,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -359,15 +359,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -388,15 +388,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -417,15 +417,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -446,15 +446,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -475,15 +475,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -504,15 +504,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -533,15 +533,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -562,15 +562,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -591,15 +591,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -620,15 +620,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -649,15 +649,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -678,15 +678,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -707,15 +707,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -736,15 +736,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -765,15 +765,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -794,15 +794,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -823,15 +823,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -852,15 +852,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -881,15 +881,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -910,15 +910,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -939,15 +939,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -968,15 +968,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -997,15 +997,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1026,15 +1026,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1055,15 +1055,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1084,15 +1084,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1113,15 +1113,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1142,15 +1142,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1171,15 +1171,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1200,15 +1200,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1229,15 +1229,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1258,15 +1258,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1287,15 +1287,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1316,15 +1316,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1345,15 +1345,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1374,15 +1374,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1403,15 +1403,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1432,15 +1432,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1461,15 +1461,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1490,15 +1490,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1519,15 +1519,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1548,15 +1548,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1577,15 +1577,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1606,15 +1606,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1635,15 +1635,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1664,15 +1664,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1693,15 +1693,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1722,15 +1722,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1751,15 +1751,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1780,15 +1780,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1809,15 +1809,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1838,15 +1838,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1867,15 +1867,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1896,15 +1896,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1925,15 +1925,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1954,15 +1954,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -1983,15 +1983,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2012,15 +2012,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2041,15 +2041,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2070,15 +2070,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2099,15 +2099,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2128,15 +2128,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2157,15 +2157,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2186,15 +2186,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2215,15 +2215,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2244,15 +2244,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2273,15 +2273,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2302,15 +2302,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2331,15 +2331,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2360,15 +2360,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2389,15 +2389,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2418,15 +2418,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2447,15 +2447,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2476,15 +2476,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2505,15 +2505,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2534,15 +2534,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2563,15 +2563,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2592,15 +2592,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2621,15 +2621,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2650,15 +2650,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2679,15 +2679,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2708,15 +2708,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2737,15 +2737,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2766,15 +2766,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2795,15 +2795,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2824,15 +2824,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2853,15 +2853,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2882,15 +2882,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2911,15 +2911,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2940,15 +2940,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2969,15 +2969,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -2998,15 +2998,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3027,15 +3027,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3056,15 +3056,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3085,15 +3085,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3114,15 +3114,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3143,15 +3143,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3172,15 +3172,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3201,15 +3201,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3230,15 +3230,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3259,15 +3259,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3288,15 +3288,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3317,15 +3317,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3346,15 +3346,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3375,15 +3375,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3404,15 +3404,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3433,15 +3433,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3462,15 +3462,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3491,15 +3491,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3520,15 +3520,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3549,15 +3549,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3578,15 +3578,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3607,15 +3607,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3636,15 +3636,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3665,15 +3665,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3694,15 +3694,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3723,15 +3723,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3752,15 +3752,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3781,15 +3781,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3810,15 +3810,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3839,15 +3839,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3868,15 +3868,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3897,15 +3897,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3926,15 +3926,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3955,15 +3955,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -3984,15 +3984,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -4013,15 +4013,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -4042,15 +4042,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -4071,15 +4071,15 @@ "operation": [ { "name": "aggregate", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/aggregate-6" }, { "name": "extract", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/extract-6" }, { "name": "fhirPath", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/search-6" } ] }, @@ -4101,15 +4101,15 @@ "operation": [ { "name": "import", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/import-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/import-6" }, { "name": "result", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/result-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/result-6" }, { "name": "job", - "definition": "https://pathling.csiro.au/fhir/OperationDefinition/job-5" + "definition": "https://pathling.csiro.au/fhir/OperationDefinition/job-6" } ] } diff --git a/pom.xml b/pom.xml index 6b6694cc74..995e6b7829 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ - 5 + 6 UTF-8 6.0.3 From ef1c64c94c30167a1f90fa9c3bc947282c52e45f Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 14:19:06 +1000 Subject: [PATCH 69/83] Revert "Add standard JVM args to benchmark execution" This reverts commit 6877729328953d0b4b3144297af6b232e985f445. --- fhir-server/pom.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 51c24ea59d..72e3710956 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -429,9 +429,6 @@ - -Xmx4g - -ea - -Duser.timezone=UTC ${project.build.directory}/benchmark From 2a6de5dcd84ce8aacdb332dcf19b15d086e8ad73 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 14:20:28 +1000 Subject: [PATCH 70/83] Add JVM arguments to @Fork annotation on benchmarks --- .../test/benchmark/AggregateBenchmark.java | 4 ++-- .../test/benchmark/DecimalBenchmark.java | 24 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java index 41ff94215b..195f5cf110 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java @@ -2,11 +2,11 @@ import static org.mockito.Mockito.mock; -import au.csiro.pathling.config.Configuration; import au.csiro.pathling.aggregate.AggregateExecutor; import au.csiro.pathling.aggregate.AggregateRequest; import au.csiro.pathling.aggregate.AggregateRequestBuilder; import au.csiro.pathling.aggregate.AggregateResponse; +import au.csiro.pathling.config.Configuration; import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.io.Database; @@ -40,7 +40,7 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Tag("UnitTest") -@Fork(0) +@Fork(jvmArgsAppend = "-Xmx4g -ea -Duser.timezone=UTC") @Warmup(iterations = 1) @Measurement(iterations = 5) public class AggregateBenchmark { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java index 1a5cfd9666..0369d97b3d 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -5,26 +5,34 @@ import au.csiro.pathling.sql.types.FlexDecimal; import au.csiro.pathling.sql.types.FlexiDecimal; import au.csiro.pathling.test.builders.DatasetBuilder; +import java.math.BigDecimal; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.Tag; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ActiveProfiles; -import javax.annotation.Nonnull; -import java.math.BigDecimal; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.mock; @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Tag("UnitTest") -@Fork(0) +@Fork(jvmArgsAppend = "-Xmx4g -ea -Duser.timezone=UTC") @Warmup(iterations = 3) @Measurement(iterations = 7) public class DecimalBenchmark { From 356101aee3914a53a852036ad66796513984c287 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 14:42:06 +1000 Subject: [PATCH 71/83] Revert "Add JVM arguments to @Fork annotation on benchmarks" This reverts commit 2a6de5dcd84ce8aacdb332dcf19b15d086e8ad73. --- .../test/benchmark/AggregateBenchmark.java | 4 ++-- .../test/benchmark/DecimalBenchmark.java | 24 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java index 195f5cf110..41ff94215b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/AggregateBenchmark.java @@ -2,11 +2,11 @@ import static org.mockito.Mockito.mock; +import au.csiro.pathling.config.Configuration; import au.csiro.pathling.aggregate.AggregateExecutor; import au.csiro.pathling.aggregate.AggregateRequest; import au.csiro.pathling.aggregate.AggregateRequestBuilder; import au.csiro.pathling.aggregate.AggregateResponse; -import au.csiro.pathling.config.Configuration; import au.csiro.pathling.encoders.FhirEncoders; import au.csiro.pathling.fhir.TerminologyServiceFactory; import au.csiro.pathling.io.Database; @@ -40,7 +40,7 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Tag("UnitTest") -@Fork(jvmArgsAppend = "-Xmx4g -ea -Duser.timezone=UTC") +@Fork(0) @Warmup(iterations = 1) @Measurement(iterations = 5) public class AggregateBenchmark { diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java index 0369d97b3d..1a5cfd9666 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -5,34 +5,26 @@ import au.csiro.pathling.sql.types.FlexDecimal; import au.csiro.pathling.sql.types.FlexiDecimal; import au.csiro.pathling.test.builders.DatasetBuilder; -import java.math.BigDecimal; -import java.util.List; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.Tag; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ActiveProfiles; +import javax.annotation.Nonnull; +import java.math.BigDecimal; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @Tag("UnitTest") -@Fork(jvmArgsAppend = "-Xmx4g -ea -Duser.timezone=UTC") +@Fork(0) @Warmup(iterations = 3) @Measurement(iterations = 7) public class DecimalBenchmark { From 459ed1f79ffc5a514c9c539bcec9a72f7010f2c3 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 14:43:13 +1000 Subject: [PATCH 72/83] Add JVM arguments to options applied within PathlingBenchmarkRunner --- .../csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java index 03df73da26..fe51ee95f5 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java @@ -59,7 +59,7 @@ public static void main(final String... argc) throws Exception { .resultFormat(resultsFileOutputType) .result(resultFile) .shouldFailOnError(true) - .jvmArgs("-server") + .jvmArgs("-server -Xmx4g -ea -Duser.timezone=UTC") .build(); new Runner(opt).run(); } From d2c85e4a7b42cc4da92b855d0813fc33493685f1 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 15:01:54 +1000 Subject: [PATCH 73/83] Increase Xmx on benchmark to 6G --- .../csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java index fe51ee95f5..f7faa6a82f 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java @@ -59,7 +59,7 @@ public static void main(final String... argc) throws Exception { .resultFormat(resultsFileOutputType) .result(resultFile) .shouldFailOnError(true) - .jvmArgs("-server -Xmx4g -ea -Duser.timezone=UTC") + .jvmArgs("-server -Xmx6g -ea -Duser.timezone=UTC") .build(); new Runner(opt).run(); } From 7f309f6055077833839f493b512a52f65d9a46ed Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 15:21:07 +1000 Subject: [PATCH 74/83] Make DecimalBenchmark rows smaller, to fit into test runner memory --- .../test/benchmark/DecimalBenchmark.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java index 1a5cfd9666..bc790f220b 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -5,21 +5,29 @@ import au.csiro.pathling.sql.types.FlexDecimal; import au.csiro.pathling.sql.types.FlexiDecimal; import au.csiro.pathling.test.builders.DatasetBuilder; +import java.math.BigDecimal; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.Tag; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ActiveProfiles; -import javax.annotation.Nonnull; -import java.math.BigDecimal; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.mock; @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) @@ -29,7 +37,7 @@ @Measurement(iterations = 7) public class DecimalBenchmark { - private static final int ROWS = 1000000; + private static final int ROWS = 100_000; private static final BigDecimal LEFT_DECIMAL = new BigDecimal( "12345678901234567890123456.123456"); private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("0.12345678901234567890123456"); From d27bbc26983ea75d1bdd080f1327c394c728bc30 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Mon, 24 Oct 2022 16:00:49 +1000 Subject: [PATCH 75/83] Add documentation for until function --- site/docs/fhirpath/functions.md | 361 +++++++++++++++++--------------- 1 file changed, 191 insertions(+), 170 deletions(-) diff --git a/site/docs/fhirpath/functions.md b/site/docs/fhirpath/functions.md index 4717c14a40..2b9fa2fb07 100644 --- a/site/docs/fhirpath/functions.md +++ b/site/docs/fhirpath/functions.md @@ -13,93 +13,53 @@ The notation used to describe the type signature of each function is as follows: [input type] -> [function name]([argument name]: [argument type], ...): [return type] ``` -## count - -``` -collection -> count() : Integer -``` - -Returns the [Integer](./data-types#integer) count of the number of items in -the input collection. - -Example: - -``` -Patient.name.given.count() -``` - -See also: [count](https://hl7.org/fhirpath/#count-integer) - -## sum - -``` -collection -> sum() : Integer|Decimal -``` - -Returns the sum of the numeric values ([Integer](./data-types#integer) or -[Decimal](./data-types#decimal)) in the input collection. - -Example: - -``` -Observation.valueDecimal.sum() -``` - -:::note -The `sum` function is not within the FHIRPath specification, and is currently -unique to the Pathling implementation. -::: - -## first +## allFalse ``` -collection -> first() : collection +collection -> allFalse() : Boolean ``` -Returns a collection containing only the first item in the input collection. -This function will return an empty collection if the input collection has no -items. +Takes a collection of Boolean values and returns `true` if all the items are `false`. If any items are `true`, the result is `false`. If the input is empty (`{ }`), the result is `true`. Example: ``` -Patient.name.given.first() +Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').allFalse() ``` -See also: [first](https://hl7.org/fhirpath/#first-collection) +See also: [allFalse](https://hl7.org/fhirpath/#allfalse-boolean) -## empty +## allTrue ``` -collection -> empty() : Boolean +collection -> allTrue() : Boolean ``` -Returns `true` if the input collection is empty, and `false` otherwise. +Takes a collection of Boolean values and returns `true` if all the items are `true`. If any items are `false`, the result is `false`. If the input is empty (`{ }`), the result is `true`. Example: ``` -Patient.reverseResolve(Condition.subject).empty() +Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').allTrue() ``` -See also: [empty](https://hl7.org/fhirpath/#empty-boolean) +See also: [allTrue](https://hl7.org/fhirpath/#alltrue-boolean) -## not +## anyFalse ``` -Boolean -> not() : Boolean +collection -> anyFalse() : Boolean ``` -Returns `true` if the input collection evaluates to `false`, and `false` if it -evaluates to `true`. Otherwise, the result is empty (`{ }`). +Takes a collection of Boolean values and returns `true` if any of the items are `false`. If all the items are `true`, or if the input is empty (`{ }`), the result is `false`. Example: ``` -(Patient.name.given contains 'Frank').not() +Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').anyFalse() ``` -See also: [not](http://hl7.org/fhirpath/#not-boolean) +See also: [anyFalse](https://hl7.org/fhirpath/#anyfalse-boolean) ## anyTrue @@ -117,78 +77,77 @@ Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105 See also: [anyTrue](https://hl7.org/fhirpath/#anytrue-boolean) -## anyFalse +## count ``` -collection -> anyFalse() : Boolean +collection -> count() : Integer ``` -Takes a collection of Boolean values and returns `true` if any of the items are `false`. If all the items are `true`, or if the input is empty (`{ }`), the result is `false`. +Returns the [Integer](./data-types#integer) count of the number of items in +the input collection. Example: ``` -Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').anyFalse() +Patient.name.given.count() ``` -See also: [anyFalse](https://hl7.org/fhirpath/#anyfalse-boolean) +See also: [count](https://hl7.org/fhirpath/#count-integer) -## allTrue +## empty ``` -collection -> allTrue() : Boolean +collection -> empty() : Boolean ``` -Takes a collection of Boolean values and returns `true` if all the items are `true`. If any items are `false`, the result is `false`. If the input is empty (`{ }`), the result is `true`. +Returns `true` if the input collection is empty, and `false` otherwise. Example: ``` -Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').allTrue() +Patient.reverseResolve(Condition.subject).empty() ``` -See also: [allTrue](https://hl7.org/fhirpath/#alltrue-boolean) +See also: [empty](https://hl7.org/fhirpath/#empty-boolean) -## allFalse +## extension ``` -collection -> allFalse() : Boolean +[any] -> extension(url: string) : collection ``` -Takes a collection of Boolean values and returns `true` if all the items are `false`. If any items are `true`, the result is `false`. If the input is empty (`{ }`), the result is `true`. - -Example: +Will filter the input collection for items named `extension` with the given url. +This is a syntactical shortcut for `.extension.where(url = string)`, but is +simpler to write. Will return an empty collection if the input collection is +empty or the url is empty. -``` -Condition.code.memberOf('http://snomed.info/sct?fhir_vs=refset/32570581000036105').allFalse() -``` +:::tip +Your extension content will only be encoded upon import if your encoding +configuration has specified that it should be. Data types and maximum depth of +encoding are both configurable. +See [Configuration](/docs/server/configuration#encoding) for more information. +::: -See also: [allFalse](https://hl7.org/fhirpath/#allfalse-boolean) +See +also: [Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) -## where +## first ``` -collection -> where(criteria: [any]) : collection +collection -> first() : collection ``` -Returns a collection containing only those elements in the input collection for -which the `criteria` expression evaluates to `true`. Elements for which the -expression evaluates to `false` or an empty collection will return an empty -collection. - -The `$this` keyword can be used within the criteria expression to refer to the -item from the input collection currently under evaluation. The context inside -the arguments is also set to the current item, so paths from the root are -assumed to be path traversals from the current element. +Returns a collection containing only the first item in the input collection. +This function will return an empty collection if the input collection has no +items. Example: ``` -Patient.reverseResolve(Condition.subject).where(recordedDate > @1960).severity +Patient.name.given.first() ``` -See also: -[where](https://hl7.org/fhirpath/#wherecriteria-expression-collection) +See also: [first](https://hl7.org/fhirpath/#first-collection) ## iif @@ -196,8 +155,8 @@ See also: [any] -> iif(condition: Boolean, ifTrue: [any], otherwise: [any]) : [any] ``` -Takes three arguments, the first of which is a Boolean expression. Returns the -second argument if the first argument evaluates to `true`, or the third argument +Takes three arguments, the first of which is a Boolean expression. Returns the +second argument if the first argument evaluates to `true`, or the third argument otherwise. The `ifTrue` and `otherwise` arguments must be of the same type. @@ -235,6 +194,106 @@ a configured See also: [Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) +## not + +``` +Boolean -> not() : Boolean +``` + +Returns `true` if the input collection evaluates to `false`, and `false` if it +evaluates to `true`. Otherwise, the result is empty (`{ }`). + +Example: + +``` +(Patient.name.given contains 'Frank').not() +``` + +See also: [not](http://hl7.org/fhirpath/#not-boolean) + +## ofType + +``` +collection -> ofType(type: Resource): collection +``` + +Returns a collection that contains all items in the input collection that are of +the given type. It is often necessary to use the `ofType` function in +conjunction with the `resolve` function, to resolve references that are +polymorphic. + +Example: + +``` +Condition.subject.resolve().ofType(Patient).gender +``` + +:::caution +This function is currently only supported for use with the [resolve](#resolve) +function, for the purpose of disambiguating polymorphic resource references. +::: + +See also: [ofType](https://hl7.org/fhirpath/#oftypetype-identifier-collection) + +## resolve + +``` +Reference -> resolve(): collection +``` + +The `resolve` function is used to traverse references between FHIR resources. +Given a collection of +[References](https://hl7.org/fhir/R4/references.html#Reference), this function +will return a collection of the resources to which they refer. + +Example: + +``` +AllergyIntolerance.patient.resolve().gender +``` + +:::caution +The following types of references are not currently supported: + +- References to individual technical versions of a resource +- Logical references (via identifier) +- References to contained resources +- Absolute literal references +::: + +See also: +[Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) + +## reverseResolve + +``` +collection -> reverseResolve(sourceReference: Reference): collection +``` + +In FHIR, resource references are unidirectional, and often the source of the +reference will be a resource type which is not the subject of the current path. + +The `reverseResolve` function takes a collection of Resources as input, and a +[Reference](https://hl7.org/fhir/R4/references.html#Reference) as the argument. +It returns a collection of all the parent resources of the source References +that resolve to the input resource. + +Example: + +``` +Patient.reverseResolve(Encounter.subject).reasonCode +``` + +:::note +The `reverseResolve` function is not within the FHIRPath specification, and is +currently unique to the Pathling implementation. +::: + +:::caution +In regard to types of references supported, the same caveats apply as those +described in the [resolve](#resolve) function. +::: + ## subsumes ``` @@ -271,7 +330,7 @@ collection -> subsumedBy(code: Coding|CodeableConcept) : ``` The `subsumedBy` function is the inverse of the [subsumes](#subsumes) function, -examining whether each input concept is _subsumed by_ any of the argument +examining whether each input concept is _subsumed by_ any of the argument concepts. Example: @@ -290,6 +349,26 @@ requires a configured See also: [Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) +## sum + +``` +collection -> sum() : Integer|Decimal +``` + +Returns the sum of the numeric values ([Integer](./data-types#integer) or +[Decimal](./data-types#decimal)) in the input collection. + +Example: + +``` +Observation.valueDecimal.sum() +``` + +:::note +The `sum` function is not within the FHIRPath specification, and is currently +unique to the Pathling implementation. +::: + ## translate ``` @@ -326,106 +405,48 @@ The `translate` function is not within the FHIRPath specification, and is currently unique to the Pathling implementation. ::: -## resolve +## until ``` -Reference -> resolve(): collection +DateTime|Date -> until(end: DateTime|Date, unit: String) : Integer ``` -The `resolve` function is used to traverse references between FHIR resources. -Given a collection of -[References](https://hl7.org/fhir/R4/references.html#Reference), this function -will return a collection of the resources to which they refer. +This function is used to calculate the difference between two `Date` +or `DateTime` values. The result is returned as an integer, with the unit of +time specified by the `unit` argument. Example: ``` -AllergyIntolerance.patient.resolve().gender -``` - -:::caution -The following types of references are not currently supported: - -- References to individual technical versions of a resource -- Logical references (via identifier) -- References to contained resources -- Absolute literal references -::: - -See also: -[Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) - -## reverseResolve - -``` -collection -> reverseResolve(sourceReference: Reference): collection -``` - -In FHIR, resource references are unidirectional, and often the source of the -reference will be a resource type which is not the subject of the current path. - -The `reverseResolve` function takes a collection of Resources as input, and a -[Reference](https://hl7.org/fhir/R4/references.html#Reference) as the argument. -It returns a collection of all the parent resources of the source References -that resolve to the input resource. - -Example: - -``` -Patient.reverseResolve(Encounter.subject).reasonCode +period.start.until(%resource.period.end, 'minutes') ``` :::note -The `reverseResolve` function is not within the FHIRPath specification, and is +The `until` function is not within the FHIRPath specification, and is currently unique to the Pathling implementation. ::: -:::caution -The same caveats apply with regards to types of references supported as -described in the [resolve](#resolve) function. -::: - -## ofType - -``` -collection -> ofType(type: Resource): collection -``` - -Returns a collection that contains all items in the input collection that are of -the given type. It is often necessary to use the `ofType` function in -conjunction with the `resolve` function, to resolve references that are -polymorphic. - -Example: +## where ``` -Condition.subject.resolve().ofType(Patient).gender +collection -> where(criteria: [any]) : collection ``` -:::caution -This function is currently only supported for use with the [resolve](#resolve) -function, for the purpose of disambiguating polymorphic resource references. -::: +Returns a collection containing only those elements in the input collection for +which the `criteria` expression evaluates to `true`. Elements for which the +expression evaluates to `false` or an empty collection will return an empty +collection. -See also: [ofType](https://hl7.org/fhirpath/#oftypetype-identifier-collection) +The `$this` keyword can be used within the criteria expression to refer to the +item from the input collection currently under evaluation. The context inside +the arguments is also set to the current item, so paths from the root are +assumed to be path traversals from the current element. -## extension +Example: ``` -[any] -> extension(url: string) : collection +Patient.reverseResolve(Condition.subject).where(recordedDate > @1960).severity ``` -Will filter the input collection for items named `extension` with the given url. -This is a syntactical shortcut for `.extension.where(url = string)`, but is -simpler to write. Will return an empty collection if the input collection is -empty or the url is empty. - -:::tip -Your extension content will only be encoded upon import if your encoding -configuration has specified that it should be. Data types and maximum depth of -encoding are both configurable. -See [Configuration](/docs/server/configuration#encoding) for more information. -::: - -See -also: [Additional functions](https://hl7.org/fhir/R4/fhirpath.html#functions) +See also: +[where](https://hl7.org/fhirpath/#wherecriteria-expression-collection) From d9cb40ba2b0826e4b7bf20d50059cf53537a729a Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 10:28:18 +1000 Subject: [PATCH 76/83] Documentation updates: math operators, until function --- site/docs/fhirpath/data-types.md | 4 ++-- site/docs/fhirpath/operators.md | 34 ++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/site/docs/fhirpath/data-types.md b/site/docs/fhirpath/data-types.md index 28ac086bd7..49dae3d6f6 100644 --- a/site/docs/fhirpath/data-types.md +++ b/site/docs/fhirpath/data-types.md @@ -209,9 +209,9 @@ unique to the Pathling implementation. ## Materializable types There is a subset of all possible FHIR types that can be "materialized", i.e. -used as the result of a grouping expression in +used as the result of an aggregation or grouping expression in the [aggregate](/docs/server/operations/aggregate) -operation, or the definition of a column within +operation, or a column expression within the [extract](/docs/server/operations/extract) operation. These types are: diff --git a/site/docs/fhirpath/operators.md b/site/docs/fhirpath/operators.md index a802250033..364601c95a 100644 --- a/site/docs/fhirpath/operators.md +++ b/site/docs/fhirpath/operators.md @@ -39,10 +39,10 @@ individual characters. All comparison operators return a [Boolean](/docs/fhirpath/data-types#boolean) value. * Not all Quantity values are comparable, it depends upon the -comparability of the units. See the -[FHIRPath specification](https://hl7.org/fhirpath/#comparison) for details on -how Quantity values are compared. Quantities with a `sqlComparator` are treated as -not comparable by this implementation. +comparability of the units. See +the FHIRPath specification for +details on how Quantity values are compared. +

See also: [Comparison](https://hl7.org/fhirpath/#comparison) @@ -62,8 +62,7 @@ an empty collection. Not all Quantity, Date and DateTime values can be compared for equality, it depends upon the comparability of the units within the Quantity values. See the [FHIRPath specification](https://hl7.org/fhirpath/#quantity-equality) for -details on how equality works with Quantity values. Quantities with a -`sqlComparator` are treated as not comparable by this implementation. +details on how equality works with Quantity values. See also: [Equality](https://hl7.org/fhirpath/#equality) @@ -77,19 +76,28 @@ The following math operators are supported: - `/` - Division - `mod` - Modulus -Math operators support only [Integer](/docs/fhirpath/data-types#integer) and -[Decimal](/docs/fhirpath/data-types#decimal) operands. - -The type of the two operands can be mixed. `+`, `-` and `*` return the same type -as the left operand, `/` returns [Decimal](/docs/fhirpath/data-types#decimal) and `mod` -returns [Integer](/docs/fhirpath/data-types#integer). +Math operators support [Integer](/docs/fhirpath/data-types#integer), +[Decimal](/docs/fhirpath/data-types#decimal) +and [Quantity](/docs/fhirpath/data-types#quantity) operands. The modulus +operator is not supported for Quantity types. Both operands must be singular. If one or both of the operands is an empty collection, the operator will return an empty collection. -See also: [Math](https://hl7.org/fhirpath/#math) +Integer and Decimal types can be mixed, while Quantity types can only be used +with other Quantity types. + +For Integer and Decimal, `+`, `-` and `*` return the same type as the left +operand, `/` returns [Decimal](/docs/fhirpath/data-types#decimal) and `mod` +returns [Integer](/docs/fhirpath/data-types#integer). + +For Quantity types, math operators return a new Quantity with the canonical unit +common to both operands. If the units are not comparable, an empty collection is +returned. + +See also: [Math](https://hl7.org/fhirpath/#math-2) ## Date/time arithmetic From 55dfb4ebcc8798f26ccc7555c7e35585b053be66 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 10:29:07 +1000 Subject: [PATCH 77/83] Add test and validation for comparable operands to math operators --- .../fhirpath/operator/MathOperator.java | 9 ++++++ .../operator/MathOperatorValidationTest.java | 31 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java index a1ed24eef6..2e3891354b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/MathOperator.java @@ -11,6 +11,7 @@ import static au.csiro.pathling.utilities.Preconditions.checkUserInput; import au.csiro.pathling.QueryHelpers.JoinType; +import au.csiro.pathling.fhirpath.Comparable; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Numeric; import au.csiro.pathling.fhirpath.Numeric.MathOperation; @@ -58,6 +59,14 @@ public FhirPath invoke(@Nonnull final OperatorInput input) { "Left operand to " + type + " operator must be singular: " + left.getExpression()); checkUserInput(right.isSingular(), "Right operand to " + type + " operator must be singular: " + right.getExpression()); + checkUserInput(left instanceof Comparable && right instanceof Comparable, + "Left and right operands are not comparable: " + left.getExpression() + " " + + type + " " + right.getExpression()); + final Comparable comparableLeft = (Comparable) left; + final Comparable comparableRight = (Comparable) right; + checkUserInput(comparableLeft.isComparableTo(comparableRight.getClass()), + "Left and right operands are not comparable: " + left.getExpression() + " " + + type + " " + right.getExpression()); final String expression = buildExpression(input, type.toString()); final Dataset dataset = join(input.getContext(), left, right, JoinType.LEFT_OUTER); diff --git a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorValidationTest.java b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorValidationTest.java index d56b72e47b..63b0aa7f23 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorValidationTest.java +++ b/fhir-server/src/test/java/au/csiro/pathling/fhirpath/operator/MathOperatorValidationTest.java @@ -105,4 +105,35 @@ void operandIsNotSingular() { reversedError.getMessage()); } + @Test + void operandsAreNotComparable() { + final ElementPath left = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.INTEGER) + .singular(true) + .expression("foo") + .build(); + final ElementPath right = new ElementPathBuilder(spark) + .fhirType(FHIRDefinedType.QUANTITY) + .singular(true) + .expression("bar") + .build(); + + final OperatorInput input = new OperatorInput(parserContext, left, right); + final Operator mathOperator = Operator.getInstance("+"); + final InvalidUserInputError error = assertThrows( + InvalidUserInputError.class, + () -> mathOperator.invoke(input)); + assertEquals("Left and right operands are not comparable: foo + bar", + error.getMessage()); + + // Now test the right operand. + final OperatorInput reversedInput = new OperatorInput(parserContext, right, left); + final InvalidUserInputError reversedError = assertThrows( + InvalidUserInputError.class, + () -> mathOperator.invoke(reversedInput)); + assertEquals( + "Left and right operands are not comparable: bar + foo", + reversedError.getMessage()); + } + } From f53228c3745cd0ba655ea822504d2bb386b19fa5 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 10:31:21 +1000 Subject: [PATCH 78/83] Exclude DecimalBenchmark from runner, and bring memory requirement back down to 4G --- .../csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java index f7faa6a82f..f948420231 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java @@ -46,6 +46,7 @@ public static void main(final String... argc) throws Exception { final Options opt = new OptionsBuilder() .include("\\.[^.]+Benchmark\\.[^.]+$") + .exclude("\\.[^.]+DecimalBenchmark\\.[^.]+$") .warmupIterations(warmup) .measurementIterations(iterations) // single shot for each iteration: @@ -59,7 +60,7 @@ public static void main(final String... argc) throws Exception { .resultFormat(resultsFileOutputType) .result(resultFile) .shouldFailOnError(true) - .jvmArgs("-server -Xmx6g -ea -Duser.timezone=UTC") + .jvmArgs("-server -Xmx4g -ea -Duser.timezone=UTC") .build(); new Runner(opt).run(); } From 84b57c6b5b3b9acbe185f6799366c9f562b29415 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 10:57:12 +1000 Subject: [PATCH 79/83] Fix inspections in FlexiDecimal --- .../pathling/sql/types/FlexiDecimal.java | 41 ++++++++++++------- .../fhirpath/element/QuantityPath.java | 2 +- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java index 96ee92df2e..462a53c2ce 100644 --- a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java +++ b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexiDecimal.java @@ -1,6 +1,10 @@ package au.csiro.pathling.sql.types; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import java.math.BigDecimal; +import java.math.RoundingMode; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -8,24 +12,28 @@ import org.apache.spark.sql.api.java.UDF2; import org.apache.spark.sql.expressions.UserDefinedFunction; import org.apache.spark.sql.functions; -import org.apache.spark.sql.types.*; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.math.BigDecimal; -import java.math.RoundingMode; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.MetadataBuilder; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; /** * Implementation of flexible decimal type represented as the unscaled value with up to 38 digits * and the scale. + * + * @author Piotr Szul */ public class FlexiDecimal { /** - * The maximum precision (the number of significant digits) + * The maximum precision (the number of significant digits). */ public static final int MAX_PRECISION = 38; /** - * The Sql type of for the unscaled value. + * The SQL type for the unscaled value. */ public static final DataType DECIMAL_TYPE = DataTypes.createDecimalType(MAX_PRECISION, 0); @@ -39,17 +47,18 @@ private static StructType createFlexibleDecimalType() { } /** - * The Sql (struct) type for flexible decimal + * The SQL (struct) type for flexible decimal. */ @Nonnull public static DataType DATA_TYPE = createFlexibleDecimalType(); @Nonnull private static UserDefinedFunction toBooleanUdf( - UDF2 method) { + @Nonnull final UDF2 method) { final UDF2 f = (left, right) -> { final BigDecimal leftValue = fromValue(left); final BigDecimal rightValue = fromValue(right); + //noinspection ReturnOfNull return (leftValue == null || rightValue == null) ? null : method.call(leftValue, rightValue); @@ -59,7 +68,8 @@ private static UserDefinedFunction toBooleanUdf( @Nonnull private static UDF2 wrapBigDecimal2( - UDF2 method) { + @Nonnull final UDF2 method) { + //noinspection ReturnOfNull return (left, right) -> (left == null || right == null) ? null @@ -68,7 +78,7 @@ private static UDF2 wrapBigDecimal2( @Nonnull private static UserDefinedFunction toBigDecimalUdf( - UDF2 method) { + @Nonnull final UDF2 method) { return functions.udf(wrapBigDecimal2(method), DATA_TYPE); } @@ -76,7 +86,7 @@ private static UserDefinedFunction toBigDecimalUdf( * Decodes a flexible decimal from the Row * * @param row the row to decode - * @return the BigDecmal representation of the row + * @return the BigDecimal representation of the row */ @Nullable public static BigDecimal fromValue(@Nullable final Row row) { @@ -101,7 +111,7 @@ public static Row toValue(@Nullable final BigDecimal decimal) { @Nullable private static Object[] toArrayValue(@Nullable final BigDecimal decimal) { - BigDecimal normalizedValue = normalize(decimal); + final BigDecimal normalizedValue = normalize(decimal); return normalizedValue != null ? new Object[]{Decimal.apply(normalizedValue.unscaledValue()), normalizedValue.scale()} : null; @@ -118,7 +128,8 @@ public static BigDecimal normalize(@Nullable final BigDecimal decimal) { // This may be may have too many digits if (adjustedValue.precision() > MAX_PRECISION) { // we need to adjust the scale to fit into the desired precision - int desiredScale = adjustedValue.scale() - (adjustedValue.precision() - MAX_PRECISION); + final int desiredScale = + adjustedValue.scale() - (adjustedValue.precision() - MAX_PRECISION); if (desiredScale >= 0) { return adjustedValue.setScale(desiredScale, RoundingMode.HALF_UP); } else { @@ -191,7 +202,7 @@ public static Column divide(@Nonnull final Column left, @Nonnull final Column ri } @Nonnull - public static Column to_decimal(@Nonnull final Column flexiDecimal) { + public static Column toDecimal(@Nonnull final Column flexiDecimal) { return TO_DECIMAL.apply(flexiDecimal); } } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java index 4737952d59..6dccf18459 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/element/QuantityPath.java @@ -162,7 +162,7 @@ public static Function buildMathOperation(@Nonnull fina final Column resultStruct = QuantityEncoding.toStruct( sourceContext.getField("id"), - FlexiDecimal.to_decimal(resultColumn), + FlexiDecimal.toDecimal(resultColumn), // NOTE: This (setting value_scale to null) works because we never decode this struct to a Quantity. // The only Quantities that are decoded are calendar duration quantities parsed from literals. lit(null), From 4aa670836482a664537b8ddd1c5473439607cc07 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 10:58:01 +1000 Subject: [PATCH 80/83] Remove DecimalBenchmark and FlexDecimal --- .../csiro/pathling/sql/types/FlexDecimal.java | 117 ------------ .../test/benchmark/DecimalBenchmark.java | 171 ------------------ .../benchmark/PathlingBenchmarkRunner.java | 1 - 3 files changed, 289 deletions(-) delete mode 100644 encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java delete mode 100644 fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java diff --git a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java b/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java deleted file mode 100644 index a791a004e5..0000000000 --- a/encoders/src/main/java/au/csiro/pathling/sql/types/FlexDecimal.java +++ /dev/null @@ -1,117 +0,0 @@ -package au.csiro.pathling.sql.types; - -import java.math.BigDecimal; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.api.java.UDF2; -import org.apache.spark.sql.expressions.UserDefinedFunction; -import org.apache.spark.sql.functions; -import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.DataTypes; - -public class FlexDecimal { - - public static int MAX_PRECISION = 48; - - @Nonnull - public static DataType DATA_TYPE = DataTypes.StringType; - - @Nonnull - private static UserDefinedFunction toBooleanUdf( - UDF2 method) { - final UDF2 f = (left, right) -> - (left == null || right == null) - ? null - : method.call(fromValue(left), fromValue(right)); - return functions.udf(f, DataTypes.BooleanType); - } - - @Nonnull - private static UDF2 wrapBigDecimal2( - UDF2 method) { - return (left, right) -> - (left == null || right == null) - ? null - : toValue(method.call(fromValue(left), fromValue(right))); - } - - @Nonnull - private static UserDefinedFunction toBigDecimalUdf( - UDF2 method) { - return functions.udf(wrapBigDecimal2(method), DATA_TYPE); - } - - @Nullable - public static BigDecimal fromValue(@Nullable final String value) { - return value != null - ? new BigDecimal(value) - : null; - } - - @Nullable - public static String toValue(@Nullable final BigDecimal decimal) { - return decimal != null && decimal.precision() <= MAX_PRECISION - ? decimal.toPlainString() - : null; - } - - private static final UserDefinedFunction EQUALS_UDF = toBooleanUdf((l, r) -> l.compareTo(r) == 0); - private static final UserDefinedFunction LT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) < 0); - private static final UserDefinedFunction LTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) <= 0); - private static final UserDefinedFunction GT_UDF = toBooleanUdf((l, r) -> l.compareTo(r) > 0); - private static final UserDefinedFunction GTE_UDF = toBooleanUdf((l, r) -> l.compareTo(r) >= 0); - - private static final UserDefinedFunction PLUS_UDF = toBigDecimalUdf(BigDecimal::add); - private static final UserDefinedFunction MULTIPLY_UDF = toBigDecimalUdf(BigDecimal::multiply); - private static final UserDefinedFunction MINUS_UDF = toBigDecimalUdf(BigDecimal::subtract); - private static final UserDefinedFunction DIVIDE_UDF = toBigDecimalUdf(BigDecimal::divide); - - - @Nonnull - public static Column equals(@Nonnull final Column left, @Nonnull final Column right) { - return EQUALS_UDF.apply(left, right); - } - - @Nonnull - public static Column lt(@Nonnull final Column left, @Nonnull final Column right) { - return LT_UDF.apply(left, right); - } - - @Nonnull - public static Column lte(@Nonnull final Column left, @Nonnull final Column right) { - return LTE_UDF.apply(left, right); - } - - @Nonnull - public static Column gt(@Nonnull final Column left, @Nonnull final Column right) { - return GT_UDF.apply(left, right); - } - - @Nonnull - public static Column gte(@Nonnull final Column left, @Nonnull final Column right) { - return GTE_UDF.apply(left, right); - } - - - @Nonnull - public static Column plus(@Nonnull final Column left, @Nonnull final Column right) { - return PLUS_UDF.apply(left, right); - } - - @Nonnull - public static Column multiply(@Nonnull final Column left, @Nonnull final Column right) { - return MULTIPLY_UDF.apply(left, right); - } - - - @Nonnull - public static Column minus(@Nonnull final Column left, @Nonnull final Column right) { - return MINUS_UDF.apply(left, right); - } - - @Nonnull - public static Column divide(@Nonnull final Column left, @Nonnull final Column right) { - return DIVIDE_UDF.apply(left, right); - } -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java deleted file mode 100644 index bc790f220b..0000000000 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java +++ /dev/null @@ -1,171 +0,0 @@ -package au.csiro.pathling.test.benchmark; - -import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; -import au.csiro.pathling.jmh.AbstractJmhSpringBootState; -import au.csiro.pathling.sql.types.FlexDecimal; -import au.csiro.pathling.sql.types.FlexiDecimal; -import au.csiro.pathling.test.builders.DatasetBuilder; -import java.math.BigDecimal; -import java.util.List; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.Tag; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; - -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@Tag("UnitTest") -@Fork(0) -@Warmup(iterations = 3) -@Measurement(iterations = 7) -public class DecimalBenchmark { - - private static final int ROWS = 100_000; - private static final BigDecimal LEFT_DECIMAL = new BigDecimal( - "12345678901234567890123456.123456"); - private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("0.12345678901234567890123456"); - - @State(Scope.Benchmark) - @ActiveProfiles("unit-test") - public static class DatasetState extends AbstractJmhSpringBootState { - - @Autowired - SparkSession spark; - - Dataset dataset; - - - @Setup(Level.Trial) - public void setUp() { - - DatasetBuilder datasetBuilder = new DatasetBuilder(spark) - .withColumn("leftDecimal", DecimalCustomCoder.decimalType()) - .withColumn("rightDecimal", DecimalCustomCoder.decimalType()) - .withColumn("leftFlexDecimal", FlexDecimal.DATA_TYPE) - .withColumn("rightFlexDecimal", FlexDecimal.DATA_TYPE) - .withColumn("leftFlexiDecimal", FlexiDecimal.DATA_TYPE) - .withColumn("rightFlexiDecimal", FlexiDecimal.DATA_TYPE); - - for (int i = 0; i < ROWS; i++) { - datasetBuilder = datasetBuilder.withRow( - LEFT_DECIMAL, - RIGHT_DECIMAL, - FlexDecimal.toValue(LEFT_DECIMAL), - FlexDecimal.toValue(RIGHT_DECIMAL), - FlexiDecimal.toValue(LEFT_DECIMAL), - FlexiDecimal.toValue(RIGHT_DECIMAL) - ); - } - dataset = datasetBuilder.build().cache(); - } - - @Nonnull - Column col(String name) { - return dataset.col(name); - } - - @Nonnull - List collectQuery(Column col) { - return dataset.select(col).collectAsList(); - } - } - - @Benchmark - public void multiply_decimal_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery(ds.col("leftDecimal").multiply(ds.col("rightDecimal")))); - } - - @Benchmark - public void add_decimal_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery(ds.col("leftDecimal").plus(ds.col("rightDecimal")))); - } - - @Benchmark - public void equals_decimal_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery(ds.col("leftDecimal").equalTo(ds.col("rightDecimal")))); - } - - @Benchmark - public void lt_decimal_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery(ds.col("leftDecimal").lt(ds.col("rightDecimal")))); - } - - - @Benchmark - public void multiply_flexDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexDecimal.multiply(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); - } - - @Benchmark - public void add_flexDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexDecimal.plus(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); - } - - @Benchmark - public void equals_flexDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexDecimal.equals(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); - } - - @Benchmark - public void lt_flexDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexDecimal.lt(ds.col("leftFlexDecimal"), ds.col("rightFlexDecimal")))); - } - - @Benchmark - public void multiply_flexiDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexiDecimal.multiply(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); - } - - @Benchmark - public void add_flexiDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexiDecimal.plus(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); - } - - @Benchmark - public void equals_flexiDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexiDecimal.equals(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); - } - - @Benchmark - public void lt_flexiDec_Benchmark(final Blackhole bh, - final DatasetState ds) { - bh.consume(ds.collectQuery( - FlexiDecimal.lt(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); - } - -} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java index f948420231..fe51ee95f5 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java @@ -46,7 +46,6 @@ public static void main(final String... argc) throws Exception { final Options opt = new OptionsBuilder() .include("\\.[^.]+Benchmark\\.[^.]+$") - .exclude("\\.[^.]+DecimalBenchmark\\.[^.]+$") .warmupIterations(warmup) .measurementIterations(iterations) // single shot for each iteration: From ff55db8d7bb0825f8babad8c47142a9f498ad9a2 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 11:42:27 +1000 Subject: [PATCH 81/83] Fix inspections --- .../sql/types/FlexiDecimalSupport.scala | 6 ++++-- .../fhirpath/CalendarDurationUtils.java | 16 +++++++++++----- .../au/csiro/pathling/fhirpath/Comparable.java | 10 +++++----- .../comparison/CodingSqlComparator.java | 14 ++++++++------ .../comparison/DateTimeSqlComparator.java | 8 +++++--- .../comparison/QuantitySqlComparator.java | 18 +++++++----------- .../fhirpath/encoding/QuantityEncoding.java | 15 ++++++++------- .../operator/DateArithmeticOperator.java | 3 +-- .../sql/dates/TemporalArithmeticFunction.java | 8 +++++--- .../sql/dates/TemporalComparisonFunction.java | 5 +++++ .../sql/dates/TemporalDifferenceFunction.java | 6 ++++++ .../dates/date/DateAddDurationFunction.java | 5 +++++ .../sql/dates/date/DateArithmeticFunction.java | 5 +++++ .../date/DateSubtractDurationFunction.java | 5 +++++ .../datetime/DateTimeAddDurationFunction.java | 5 +++++ .../datetime/DateTimeArithmeticFunction.java | 5 +++++ .../datetime/DateTimeComparisonFunction.java | 5 +++++ .../dates/datetime/DateTimeEqualsFunction.java | 5 +++++ .../datetime/DateTimeGreaterThanFunction.java | 5 +++++ .../DateTimeGreaterThanOrEqualToFunction.java | 5 +++++ .../datetime/DateTimeLessThanFunction.java | 5 +++++ .../DateTimeLessThanOrEqualToFunction.java | 5 +++++ .../DateTimeSubtractDurationFunction.java | 5 +++++ .../sql/dates/time/TimeComparisonFunction.java | 5 +++++ .../sql/dates/time/TimeEqualsFunction.java | 5 +++++ .../pathling/sql/dates/time/TimeFunction.java | 5 +++++ .../dates/time/TimeGreaterThanFunction.java | 5 +++++ .../time/TimeGreaterThanOrEqualToFunction.java | 5 +++++ .../sql/dates/time/TimeLessThanFunction.java | 5 +++++ .../time/TimeLessThanOrEqualToFunction.java | 7 ++++++- 30 files changed, 161 insertions(+), 45 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala index 3ef8091a9f..0ba76c9d1d 100644 --- a/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala +++ b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala @@ -1,13 +1,15 @@ package au.csiro.pathling.sql.types import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, IsNull, Literal} import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} +import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, IsNull, Literal} import org.apache.spark.sql.functions.{lit, struct} -import org.apache.spark.sql.types.{DataTypes, Decimal, DecimalType, ObjectType} +import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType} /** * Helper class for serialization of FlexiDecimals. + * + * @author Piotr Szul */ object FlexiDecimalSupport { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java index 13cb1864e4..d221d34702 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/CalendarDurationUtils.java @@ -1,22 +1,27 @@ package au.csiro.pathling.fhirpath; +import static au.csiro.pathling.utilities.Preconditions.checkUserInput; + import com.google.common.collect.ImmutableMap; -import org.hl7.fhir.r4.model.Quantity; -import javax.annotation.Nonnull; import java.math.BigDecimal; import java.util.Calendar; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import org.hl7.fhir.r4.model.Quantity; -import static au.csiro.pathling.utilities.Preconditions.checkUserInput; - +/** + * Utility methods for working with calendar durations. + * + * @author Piotr Szul + */ public final class CalendarDurationUtils { private CalendarDurationUtils() { // Toolkit class } - + public static final String FHIRPATH_CALENDAR_DURATION_URI = "https://hl7.org/fhirpath/N1/calendar-duration"; private static final Pattern CALENDAR_DURATION_PATTERN = Pattern.compile("([0-9.]+) (\\w+)"); @@ -71,4 +76,5 @@ public static Quantity parseCalendarDuration(@Nonnull final String calendarDurat calendarDuration.setCode(keyword); return calendarDuration; } + } diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java index 791fcb9060..de18e4117b 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/Comparable.java @@ -28,7 +28,7 @@ interface SqlComparator { Column equalsTo(Column left, Column right); - default Column notEqual(Column left, Column right) { + default Column notEqual(final Column left, final Column right) { return functions.not(equalsTo(left, right)); } @@ -85,12 +85,12 @@ public Column greaterThanOrEqual(@Nonnull final Column left, @Nonnull final Colu /** * Get a function that can take two Comparable paths and return a {@link Column} that contains a - * comparison condition. The type of condition is controlled by supplying a {@link - * ComparisonOperation}. + * comparison condition. The type of condition is controlled by supplying a + * {@link ComparisonOperation}. * * @param operation The {@link ComparisonOperation} type to retrieve a comparison for - * @return A {@link Function} that takes a Comparable as its parameter, and returns a {@link - * Column} + * @return A {@link Function} that takes a Comparable as its parameter, and returns a + * {@link Column} */ @Nonnull Function getComparison(@Nonnull ComparisonOperation operation); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java index 5d1b4dc2db..da461a4ab5 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/CodingSqlComparator.java @@ -6,21 +6,23 @@ package au.csiro.pathling.fhirpath.comparison; +import static org.apache.spark.sql.functions.lit; + import au.csiro.pathling.errors.InvalidUserInputError; import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.functions; -import javax.annotation.Nonnull; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import java.util.Arrays; import java.util.List; import java.util.function.Function; - -import static org.apache.spark.sql.functions.lit; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.functions; /** * Implementation of comparator for Coding type. + * + * @author Piotr Szul */ public class CodingSqlComparator implements SqlComparator { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java index 3884847571..f52d03607a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/DateTimeSqlComparator.java @@ -3,19 +3,21 @@ import static org.apache.spark.sql.functions.callUDF; import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.sql.dates.datetime.DateTimeEqualsFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeGreaterThanOrEqualToFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanFunction; import au.csiro.pathling.sql.dates.datetime.DateTimeLessThanOrEqualToFunction; -import org.apache.spark.sql.Column; -import javax.annotation.Nonnull; import java.util.function.Function; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; /** * Implementation of comparator for the DateTime type. + * + * @author Piotr Szul */ public class DateTimeSqlComparator implements SqlComparator { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java index 6eb9bc19b8..99ab9a7bf0 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/comparison/QuantitySqlComparator.java @@ -6,22 +6,23 @@ package au.csiro.pathling.fhirpath.comparison; +import static org.apache.spark.sql.functions.when; + import au.csiro.pathling.fhirpath.Comparable; -import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.Comparable.ComparisonOperation; +import au.csiro.pathling.fhirpath.Comparable.SqlComparator; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.sql.types.FlexiDecimal; -import org.apache.spark.sql.Column; -import javax.annotation.Nonnull; import java.util.function.BiFunction; import java.util.function.Function; - -import static au.csiro.pathling.fhirpath.Comparable.STD_SQL_COMPARATOR; -import static org.apache.spark.sql.functions.when; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; /** * Implementation of comparator for the Quantity type. It uses canonicalized values and units for * comparison rather than the original values. + * + * @author Piotr Szul */ public class QuantitySqlComparator implements SqlComparator { @@ -53,11 +54,6 @@ public Column equalsTo(@Nonnull final Column left, @Nonnull final Column right) return wrap(FlexiDecimal::equals).apply(left, right); } - // @Override - // public Column notEqual(@Nonnull final Column left, @Nonnull final Column right) { - // return wrap(defaultSqlComparator::notEqual).apply(left, right); - // } - @Override public Column lessThan(@Nonnull final Column left, @Nonnull final Column right) { return wrap(FlexiDecimal::lt).apply(left, right); diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index f76355b368..8a092ff46a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -11,17 +11,17 @@ import au.csiro.pathling.encoders.QuantitySupport; import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.encoders.terminology.ucum.Ucum; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; +import au.csiro.pathling.sql.types.FlexiDecimal; +import au.csiro.pathling.sql.types.FlexiDecimalSupport; +import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import au.csiro.pathling.encoders.terminology.ucum.Ucum; -import au.csiro.pathling.fhirpath.CalendarDurationUtils; -import au.csiro.pathling.sql.types.FlexiDecimal; -import au.csiro.pathling.sql.types.FlexiDecimalSupport; -import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.Column; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -35,6 +35,8 @@ /** * Object decoders/encoders for {@link Quantity}. + * + * @author Piotr Szul */ public final class QuantityEncoding { @@ -63,7 +65,7 @@ private QuantityEncoding() { * @return the Row representation of the quantity */ @Nullable - public static Row encode(@Nullable final Quantity quantity, boolean includeScale) { + public static Row encode(@Nullable final Quantity quantity, final boolean includeScale) { if (quantity == null) { return null; } @@ -219,7 +221,6 @@ public static Column encodeLiteral(@Nonnull final Quantity quantity) { final Optional comparator = Optional.ofNullable(quantity.getComparator()); final BigDecimal value = quantity.getValue(); - // FlexDecima; final BigDecimal canonicalizedValue; final String canonicalizedCode; if (quantity.getSystem().equals(Ucum.SYSTEM_URI)) { diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java index 9432f1ea7e..236fc372d3 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/operator/DateArithmeticOperator.java @@ -11,10 +11,10 @@ import static au.csiro.pathling.utilities.Preconditions.checkUserInput; import au.csiro.pathling.QueryHelpers.JoinType; +import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.FhirPath; import au.csiro.pathling.fhirpath.Numeric.MathOperation; import au.csiro.pathling.fhirpath.Temporal; -import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.literal.QuantityLiteralPath; import javax.annotation.Nonnull; import org.apache.spark.sql.Dataset; @@ -47,7 +47,6 @@ public FhirPath invoke(@Nonnull final OperatorInput input) { checkUserInput(left instanceof Temporal, type + " operator does not support left operand: " + left.getExpression()); - // TODO: It does not seem to be strictly necessary for the right argument to be a literal. checkUserInput(right instanceof QuantityLiteralPath, type + " operator does not support right operand: " + right.getExpression()); final QuantityLiteralPath calendarDuration = (QuantityLiteralPath) right; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java index 367be3dcbf..276405297a 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalArithmeticFunction.java @@ -9,10 +9,7 @@ import au.csiro.pathling.fhirpath.CalendarDurationUtils; import au.csiro.pathling.fhirpath.encoding.QuantityEncoding; import au.csiro.pathling.sql.udf.SqlFunction2; -import com.google.common.collect.ImmutableMap; import java.math.RoundingMode; -import java.util.Calendar; -import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nonnull; @@ -23,6 +20,11 @@ import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.Quantity; +/** + * Base class for functions that perform arithmetic on temporal values. + * + * @author John Grimes + */ public abstract class TemporalArithmeticFunction implements SqlFunction2 { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java index 7341b45bb3..f1c292ce95 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalComparisonFunction.java @@ -13,6 +13,11 @@ import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DataTypes; +/** + * Base class for functions that compare temporal values. + * + * @author John Grimes + */ public abstract class TemporalComparisonFunction implements SqlFunction2 { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java index a58f85434d..4775efc1dd 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/TemporalDifferenceFunction.java @@ -23,6 +23,12 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Calculates the difference between two temporal values, returning an integer value using the + * requested unit. Used for the until function. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TemporalDifferenceFunction implements SqlFunction3 { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java index dc7d7bf81c..6d79cce4a6 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateAddDurationFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Adds a duration to a date. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateAddDurationFunction extends DateArithmeticFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java index f5dbbcedd2..6251d1bb13 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateArithmeticFunction.java @@ -10,6 +10,11 @@ import java.util.function.Function; import org.hl7.fhir.r4.model.DateType; +/** + * Base class for functions that perform arithmetic on dates. + * + * @author John Grimes + */ public abstract class DateArithmeticFunction extends TemporalArithmeticFunction { private static final long serialVersionUID = 6759548804191034570L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java index fee7980aab..957db94c53 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/date/DateSubtractDurationFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Subtracts a duration from a date. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateSubtractDurationFunction extends DateArithmeticFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java index 47eb326255..5f27afdf93 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeAddDurationFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Adds a duration to a datetime. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeAddDurationFunction extends DateTimeArithmeticFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java index cdeb9b2ffb..e0e046462f 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeArithmeticFunction.java @@ -10,6 +10,11 @@ import java.util.function.Function; import org.hl7.fhir.r4.model.DateTimeType; +/** + * Base class for functions that perform arithmetic on datetimes. + * + * @author John Grimes + */ public abstract class DateTimeArithmeticFunction extends TemporalArithmeticFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java index e6b0cc10dd..db30c0b2da 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeComparisonFunction.java @@ -10,6 +10,11 @@ import java.util.function.Function; import org.hl7.fhir.r4.model.DateTimeType; +/** + * Base class for functions that compare datetimes. + * + * @author John Grimes + */ public abstract class DateTimeComparisonFunction extends TemporalComparisonFunction { private static final long serialVersionUID = -2449192480093120211L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java index 5901be60a2..bec12a402c 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeEqualsFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines the equality of two datetimes. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeEqualsFunction extends DateTimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java index b5ff30d9e4..e8c993df05 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanFunction.java @@ -13,6 +13,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one datetime is after another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeGreaterThanFunction extends DateTimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java index 29d3b19556..63fe7eca61 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeGreaterThanOrEqualToFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one datetime is after or at the same time as another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeGreaterThanOrEqualToFunction extends DateTimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java index 310c0a06eb..4441f6aefb 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanFunction.java @@ -13,6 +13,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one datetime is before another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeLessThanFunction extends DateTimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java index d10964a121..1df0d1ad43 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeLessThanOrEqualToFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one datetime is before or at the same time as another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeLessThanOrEqualToFunction extends DateTimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java index 1e360c2b2f..7e044acdbd 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/datetime/DateTimeSubtractDurationFunction.java @@ -12,6 +12,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Subtracts a duration from a datetime. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class DateTimeSubtractDurationFunction extends DateTimeArithmeticFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java index c64098e779..a169f1a3af 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeComparisonFunction.java @@ -10,6 +10,11 @@ import java.time.LocalTime; import java.util.function.Function; +/** + * Base class for functions that compare times. + * + * @author John Grimes + */ public abstract class TimeComparisonFunction extends TemporalComparisonFunction { private static final long serialVersionUID = 3661335567427062952L; diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java index 0a02d46729..b3aa590dda 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeEqualsFunction.java @@ -11,6 +11,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines the equality of two times. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TimeEqualsFunction extends TimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java index 39198bafdb..e92b61a392 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeFunction.java @@ -10,6 +10,11 @@ import java.util.regex.Pattern; import javax.annotation.Nonnull; +/** + * Common functionality relating to time operations. + * + * @author John Grimes + */ abstract class TimeFunction { private static final Pattern HOURS_ONLY = Pattern.compile("^\\d{2}$"); diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java index 5a05680636..bbc8ca1ca5 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanFunction.java @@ -11,6 +11,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one time is after another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TimeGreaterThanFunction extends TimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java index 2dd4315f53..c1f5a0462d 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeGreaterThanOrEqualToFunction.java @@ -11,6 +11,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one time is after or at the same time as another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TimeGreaterThanOrEqualToFunction extends TimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java index f59c79c9b3..0a6ae24d32 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanFunction.java @@ -11,6 +11,11 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one time is before another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TimeLessThanFunction extends TimeComparisonFunction { diff --git a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java index b5a529b9d0..7e85ac82e9 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java +++ b/fhir-server/src/main/java/au/csiro/pathling/sql/dates/time/TimeLessThanOrEqualToFunction.java @@ -11,12 +11,17 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +/** + * Determines whether one time is before or at the same time as another. + * + * @author John Grimes + */ @Component @Profile("core | unit-test") public class TimeLessThanOrEqualToFunction extends TimeComparisonFunction { private static final long serialVersionUID = -5929640258789711609L; - + public static final String FUNCTION_NAME = "time_lte"; @Override From fec855240d42241d8f97324e21f728f308612405 Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 11:42:56 +1000 Subject: [PATCH 82/83] Reinstate DecimalBenchmark, disabled and with FlexDecimal removed --- .../test/benchmark/DecimalBenchmark.java | 148 ++++++++++++++++++ .../benchmark/PathlingBenchmarkRunner.java | 1 + 2 files changed, 149 insertions(+) create mode 100644 fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java new file mode 100644 index 0000000000..7b0770e6cc --- /dev/null +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/DecimalBenchmark.java @@ -0,0 +1,148 @@ +/* + * Copyright © 2018-2022, Commonwealth Scientific and Industrial Research + * Organisation (CSIRO) ABN 41 687 119 230. Licensed under the CSIRO Open Source + * Software Licence Agreement. + */ + +package au.csiro.pathling.test.benchmark; + +import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder; +import au.csiro.pathling.jmh.AbstractJmhSpringBootState; +import au.csiro.pathling.sql.types.FlexiDecimal; +import au.csiro.pathling.test.builders.DatasetBuilder; +import java.math.BigDecimal; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.Tag; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ActiveProfiles; + +/** + * Benchmarks for the {@link FlexiDecimal} class. + * + * @author Piotr Szul + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Tag("UnitTest") +@Fork(0) +@Warmup(iterations = 3) +@Measurement(iterations = 7) +public class DecimalBenchmark { + + private static final int ROWS = 100_000; + private static final BigDecimal LEFT_DECIMAL = new BigDecimal( + "12345678901234567890123456.123456"); + private static final BigDecimal RIGHT_DECIMAL = new BigDecimal("0.12345678901234567890123456"); + + @State(Scope.Benchmark) + @ActiveProfiles("unit-test") + public static class DatasetState extends AbstractJmhSpringBootState { + + @Autowired + SparkSession spark; + + Dataset dataset; + + + @Setup(Level.Trial) + public void setUp() { + + DatasetBuilder datasetBuilder = new DatasetBuilder(spark) + .withColumn("leftDecimal", DecimalCustomCoder.decimalType()) + .withColumn("rightDecimal", DecimalCustomCoder.decimalType()) + .withColumn("leftFlexiDecimal", FlexiDecimal.DATA_TYPE) + .withColumn("rightFlexiDecimal", FlexiDecimal.DATA_TYPE); + + for (int i = 0; i < ROWS; i++) { + datasetBuilder = datasetBuilder.withRow( + LEFT_DECIMAL, + RIGHT_DECIMAL, + FlexiDecimal.toValue(LEFT_DECIMAL), + FlexiDecimal.toValue(RIGHT_DECIMAL) + ); + } + dataset = datasetBuilder.build().cache(); + } + + @Nonnull + Column col(@Nonnull final String name) { + return dataset.col(name); + } + + @Nonnull + List collectQuery(@Nonnull final Column col) { + return dataset.select(col).collectAsList(); + } + } + + @Benchmark + public void multiply_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").multiply(ds.col("rightDecimal")))); + } + + @Benchmark + public void add_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").plus(ds.col("rightDecimal")))); + } + + @Benchmark + public void equals_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").equalTo(ds.col("rightDecimal")))); + } + + @Benchmark + public void lt_decimal_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery(ds.col("leftDecimal").lt(ds.col("rightDecimal")))); + } + + @Benchmark + public void multiply_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.multiply(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + + @Benchmark + public void add_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.plus(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + + @Benchmark + public void equals_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.equals(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + + @Benchmark + public void lt_flexiDec_Benchmark(final Blackhole bh, + final DatasetState ds) { + bh.consume(ds.collectQuery( + FlexiDecimal.lt(ds.col("leftFlexiDecimal"), ds.col("rightFlexiDecimal")))); + } + +} diff --git a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java index fe51ee95f5..f948420231 100644 --- a/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java +++ b/fhir-server/src/test/java/au/csiro/pathling/test/benchmark/PathlingBenchmarkRunner.java @@ -46,6 +46,7 @@ public static void main(final String... argc) throws Exception { final Options opt = new OptionsBuilder() .include("\\.[^.]+Benchmark\\.[^.]+$") + .exclude("\\.[^.]+DecimalBenchmark\\.[^.]+$") .warmupIterations(warmup) .measurementIterations(iterations) // single shot for each iteration: From f12660d56175343cb72e15864b67a577ae72dd0c Mon Sep 17 00:00:00 2001 From: John Grimes Date: Tue, 25 Oct 2022 11:54:28 +1000 Subject: [PATCH 83/83] Fix trailing white space --- .../csiro/pathling/sql/types/FlexiDecimalSupport.scala | 10 +++++----- .../pathling/fhirpath/encoding/QuantityEncoding.java | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala index 0ba76c9d1d..2fb40ecd16 100644 --- a/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala +++ b/encoders/src/main/scala/au/csiro/pathling/sql/types/FlexiDecimalSupport.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.functions.{lit, struct} import org.apache.spark.sql.types.{DataTypes, Decimal, ObjectType} /** - * Helper class for serialization of FlexiDecimals. + * Helper class for serialization of FlexiDecimals. * * @author Piotr Szul */ @@ -27,13 +27,13 @@ object FlexiDecimalSupport { } def createFlexiDecimalSerializer(expression: Expression): Expression = { - // the expression is of type BigDecimal + // the expression is of type BigDecimal // we want to encode it as a struct of two fields // value -> the BigInteger representing digits - // scale -> the actual position of the decimal coma + // scale -> the actual position of the decimal point - // TODO: Performance: consider if the normalized value could be cached in - // a variable + // TODO: Performance: consider if the normalized value could be cached in + // a variable val normalizedExpression: Expression = StaticInvoke(classOf[FlexiDecimal], ObjectType(classOf[java.math.BigDecimal]), "normalize", diff --git a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java index 8a092ff46a..dddc99ba57 100644 --- a/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java +++ b/fhir-server/src/main/java/au/csiro/pathling/fhirpath/encoding/QuantityEncoding.java @@ -82,10 +82,9 @@ public static Row encode(@Nullable final Quantity quantity, final boolean includ } final String comparator = Optional.ofNullable(quantity.getComparator()) .map(QuantityComparator::toCode).orElse(null); - // TODO: The null scale support it a temporary measure because we currently - // cannot encode the scale of the results of arithmetic operations. return RowFactory.create(quantity.getId(), quantity.getValue(), + // We cannot encode the scale of the results of arithmetic operations. includeScale ? quantity.getValue().scale() : null,