diff --git a/README.md b/README.md index 1e5614a..77bf2eb 100644 --- a/README.md +++ b/README.md @@ -101,18 +101,29 @@ ion-java-benchmark write --ion-imports-for-benchmark tables.ion \ Benchmark a full-traversal read of data equivalent to exampleWithImports.10n, which declares the shared symbol table imports provided by inputTables.ion, re-encoded (if necessary) using the shared symbol tables provided by benchmarkTables.ion, inputTables.ion, and no shared symbol tables. Produce -results from using both the DOM and IonReader APIs. +results from using the DOM, IonReader and IonElement APIs. ``` ion-java-benchmark read --ion-imports-for-input inputTables.ion \ --ion-imports-for-benchmark benchmarkTables.ion \ --ion-imports-for-benchmark auto \ --ion-imports-for-benchmark none \ - --ion-api dom \ - --ion-api streaming \ + --api dom \ + --api streaming \ + --api ion_element_dom \ exampleWithImports.10n ``` +Benchmark a full-traversal read of `example.10n` using the IonElement API from ion-element-kotlin, +comparing performance against the traditional DOM API. + +``` +ion-java-benchmark read --api dom \ + --api ion_element_dom \ + example.10n +``` + + ## Tips As the JMH output warns: "Do not assume the numbers tell you what you want them to tell." Benchmarking diff --git a/pom.xml b/pom.xml index 567b428..19747bf 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,21 @@ ion-java-path-extraction 1.2.0 + + com.amazon.ion + ion-element + [1.3.0, ) + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + 1.6.20 + + + org.jetbrains.kotlinx + kotlinx-collections-immutable-jvm + 0.3.4 + junit @@ -117,6 +132,13 @@ 8 8 + + + org.openjdk.jmh + jmh-generator-annprocess + 1.23 + + diff --git a/src/com/amazon/ion/benchmark/API.java b/src/com/amazon/ion/benchmark/API.java index 45d8d1f..d312fb2 100644 --- a/src/com/amazon/ion/benchmark/API.java +++ b/src/com/amazon/ion/benchmark/API.java @@ -13,5 +13,10 @@ enum API { /** * For Ion: the DOM APIs (IonLoader, IonValue, etc.). For JSON (Jackson): JsonNode via ObjectMapper. */ - DOM + DOM, + + /** + * For Ion: the IonElement APIs from ion-element-kotlin. For JSON (Jackson): Not supported. + */ + ION_ELEMENT_DOM } diff --git a/src/com/amazon/ion/benchmark/CborJacksonMeasurableReadTask.java b/src/com/amazon/ion/benchmark/CborJacksonMeasurableReadTask.java index 5f1d718..a391420 100644 --- a/src/com/amazon/ion/benchmark/CborJacksonMeasurableReadTask.java +++ b/src/com/amazon/ion/benchmark/CborJacksonMeasurableReadTask.java @@ -156,4 +156,14 @@ void fullyReadDomFromFile(SideEffectConsumer consumer) throws IOException { consumer.consume(iterator.next()); } } + + @Override + public void fullyReadElementFromBuffer(SideEffectConsumer consumer) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for CBOR format. Use ion_binary or ion_text format instead."); + } + + @Override + public void fullyReadElementFromFile(SideEffectConsumer consumer) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for CBOR format. Use ion_binary or ion_text format instead."); + } } diff --git a/src/com/amazon/ion/benchmark/CborJacksonMeasurableWriteTask.java b/src/com/amazon/ion/benchmark/CborJacksonMeasurableWriteTask.java index bddd5b5..c942145 100644 --- a/src/com/amazon/ion/benchmark/CborJacksonMeasurableWriteTask.java +++ b/src/com/amazon/ion/benchmark/CborJacksonMeasurableWriteTask.java @@ -150,4 +150,9 @@ CBORGenerator newWriter(OutputStream outputStream) throws IOException { void closeWriter(CBORGenerator generator) throws IOException { generator.close(); } + + @Override + void generateWriteInstructionsElement(Consumer> instructionsSink) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for CBOR format. Use ion_binary or ion_text format instead."); + } } diff --git a/src/com/amazon/ion/benchmark/IonMeasurableReadTask.java b/src/com/amazon/ion/benchmark/IonMeasurableReadTask.java index b954246..69ce341 100644 --- a/src/com/amazon/ion/benchmark/IonMeasurableReadTask.java +++ b/src/com/amazon/ion/benchmark/IonMeasurableReadTask.java @@ -7,6 +7,9 @@ import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ionpathextraction.PathExtractor; import com.amazon.ionpathextraction.PathExtractorBuilder; +import com.amazon.ionelement.api.AnyElement; +import com.amazon.ionelement.api.IonElementLoader; +import com.amazon.ionelement.api.ElementLoader; import java.io.IOException; import java.nio.file.Path; @@ -25,6 +28,7 @@ class IonMeasurableReadTask extends MeasurableReadTask { private final byte[] reusableLobBuffer; private IonReaderBuilder readerBuilder; private SideEffectConsumer sideEffectConsumer = null; + private IonElementLoader elementLoader; /** * Returns the next power of two greater than or equal to the given value. @@ -67,6 +71,11 @@ private int pathExtractorCallback(IonReader reader) { } else { reusableLobBuffer = null; } + + // Initialize IonElement loader for ION_ELEMENT_DOM API + if (options.api == API.ION_ELEMENT_DOM) { + elementLoader = ElementLoader.createIonElementLoader(); + } } @Override @@ -240,7 +249,9 @@ public void traverseFromFile(List paths, SideEffectConsumer consumer) th public void fullyReadDomFromBuffer(SideEffectConsumer consumer) throws IOException { sideEffectConsumer = consumer; IonReader reader = readerBuilder.build(buffer); - ionSystem.newLoader().load(reader); + for (com.amazon.ion.IonValue value : ionSystem.newLoader().load(reader)) { + consumer.consume(value); + } reader.close(); } @@ -248,7 +259,31 @@ public void fullyReadDomFromBuffer(SideEffectConsumer consumer) throws IOExcepti public void fullyReadDomFromFile(SideEffectConsumer consumer) throws IOException { sideEffectConsumer = consumer; IonReader reader = readerBuilder.build(options.newInputStream(inputFile)); - ionSystem.newLoader().load(reader); + for (com.amazon.ion.IonValue value : ionSystem.newLoader().load(reader)) { + consumer.consume(value); + } + reader.close(); + } + + @Override + public void fullyReadElementFromBuffer(SideEffectConsumer consumer) throws IOException { + sideEffectConsumer = consumer; + IonReader reader = readerBuilder.build(buffer); + Iterable elements = elementLoader.loadAllElements(reader); + for (AnyElement element : elements) { + consumer.consume(element); + } + reader.close(); + } + + @Override + public void fullyReadElementFromFile(SideEffectConsumer consumer) throws IOException { + sideEffectConsumer = consumer; + IonReader reader = readerBuilder.build(options.newInputStream(inputFile)); + Iterable elements = elementLoader.loadAllElements(reader); + for (AnyElement element : elements) { + consumer.consume(element); + } reader.close(); } } diff --git a/src/com/amazon/ion/benchmark/IonMeasurableWriteTask.java b/src/com/amazon/ion/benchmark/IonMeasurableWriteTask.java index d439ef7..3a2faf6 100644 --- a/src/com/amazon/ion/benchmark/IonMeasurableWriteTask.java +++ b/src/com/amazon/ion/benchmark/IonMeasurableWriteTask.java @@ -7,13 +7,18 @@ import com.amazon.ion.IonWriter; import com.amazon.ion.SymbolToken; import com.amazon.ion.Timestamp; +import com.amazon.ionelement.api.AnyElement; +import com.amazon.ionelement.api.IonElementLoader; +import com.amazon.ionelement.api.ElementLoader; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; import static com.amazon.ion.benchmark.Constants.ION_SYSTEM; @@ -24,6 +29,7 @@ class IonMeasurableWriteTask extends MeasurableWriteTask { private final IonUtilities.IonWriterSupplier writerBuilder; + private IonElementLoader elementLoader; /** * @param inputPath path to the data to re-write. @@ -39,6 +45,11 @@ class IonMeasurableWriteTask extends MeasurableWriteTask { } else { throw new IllegalStateException("IonFormatWriter is compatible only with ION_TEXT and ION_BINARY"); } + + // Initialize IonElement loader for ION_ELEMENT_DOM API + if (options.api == API.ION_ELEMENT_DOM) { + elementLoader = ElementLoader.createIonElementLoader(); + } } /** @@ -195,4 +206,33 @@ public void closeWriter(IonWriter writer) throws IOException { writer.close(); } + @Override + void generateWriteInstructionsElement(Consumer> instructionsSink) throws IOException { + Iterable elements; + if (options.limit == Integer.MAX_VALUE) { + try (IonReader reader = IonUtilities.newReaderBuilderForInput(options).build(options.newInputStream(inputFile))) { + elements = elementLoader.loadAllElements(reader); + } + } else { + List limitedElements = new ArrayList<>(); + try (IonReader reader = IonUtilities.newReaderBuilderForInput(options).build(options.newInputStream(inputFile))) { + int count = 0; + while (count++ < options.limit && reader.next() != null) { + limitedElements.add(elementLoader.loadCurrentElement(reader)); + } + } + elements = limitedElements; + } + + // Convert IonElements to write instructions + int elementCount = 0; + for (AnyElement element : elements) { + instructionsSink.accept(element::writeTo); + elementCount++; + if (options.flushPeriod != null && elementCount % options.flushPeriod == 0) { + instructionsSink.accept(IonWriter::flush); + } + } + instructionsSink.accept(IonWriter::finish); + } } diff --git a/src/com/amazon/ion/benchmark/JsonJacksonMeasurableReadTask.java b/src/com/amazon/ion/benchmark/JsonJacksonMeasurableReadTask.java index 2cc066c..cf201f2 100644 --- a/src/com/amazon/ion/benchmark/JsonJacksonMeasurableReadTask.java +++ b/src/com/amazon/ion/benchmark/JsonJacksonMeasurableReadTask.java @@ -145,4 +145,14 @@ void fullyReadDomFromFile(SideEffectConsumer consumer) throws IOException { consumer.consume(iterator.next()); } } + + @Override + public void fullyReadElementFromBuffer(SideEffectConsumer consumer) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for JSON format. Use ion_binary or ion_text format instead."); + } + + @Override + public void fullyReadElementFromFile(SideEffectConsumer consumer) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for JSON format. Use ion_binary or ion_text format instead."); + } } diff --git a/src/com/amazon/ion/benchmark/JsonJacksonMeasurableWriteTask.java b/src/com/amazon/ion/benchmark/JsonJacksonMeasurableWriteTask.java index 0cd5c9a..47ba546 100644 --- a/src/com/amazon/ion/benchmark/JsonJacksonMeasurableWriteTask.java +++ b/src/com/amazon/ion/benchmark/JsonJacksonMeasurableWriteTask.java @@ -142,4 +142,9 @@ JsonGenerator newWriter(OutputStream outputStream) throws IOException { void closeWriter(JsonGenerator generator) throws IOException { generator.close(); } + + @Override + void generateWriteInstructionsElement(Consumer> instructionsSink) throws IOException { + throw new UnsupportedOperationException("IonElement API is not supported for JSON format. Use ion_binary or ion_text format instead."); + } } diff --git a/src/com/amazon/ion/benchmark/Main.java b/src/com/amazon/ion/benchmark/Main.java index f4fbf7b..781f4f3 100644 --- a/src/com/amazon/ion/benchmark/Main.java +++ b/src/com/amazon/ion/benchmark/Main.java @@ -133,10 +133,10 @@ public class Main { + " -f --format Format to benchmark, from the set (ion_binary | ion_text | json | " + "cbor). May be specified multiple times to compare different formats. [default: ion_binary]\n" - + " -a --api The API to exercise (dom or streaming). For the ion-binary or " + + " -a --api The API to exercise (dom, streaming, or ion_element_dom). For the ion-binary or " + "ion-text formats, 'streaming' causes IonReader/IonWriter to be used while 'dom' causes IonLoader to be " - + "used. For Jackson JSON, 'streaming' causes JsonParser/JsonGenerator to be used while 'dom' causes " - + "ObjectMapper to materialize JsonNode instances. May be specified multiple times to compare both " + + "used, and 'ion_element_dom' causes IonElement from ion-element-kotlin to be used with dedicated code paths. For Jackson JSON, 'streaming' causes JsonParser/JsonGenerator to be used while 'dom' causes " + + "ObjectMapper to materialize JsonNode instances. 'ion_element_dom' is not supported for JSON or CBOR formats. May be specified multiple times to compare " + "APIs. [default: streaming]\n" + " -I --ion-imports-for-input A file containing a sequence of Ion symbol tables, or the string " diff --git a/src/com/amazon/ion/benchmark/MeasurableReadTask.java b/src/com/amazon/ion/benchmark/MeasurableReadTask.java index d77bf75..59e5003 100644 --- a/src/com/amazon/ion/benchmark/MeasurableReadTask.java +++ b/src/com/amazon/ion/benchmark/MeasurableReadTask.java @@ -84,6 +84,20 @@ abstract class MeasurableReadTask implements MeasurableTask { */ abstract void fullyReadDomFromFile(SideEffectConsumer consumer) throws IOException; + /** + * Initialize the element loader and perform a fully-materialized deep read of the data from a buffer using + * IonElement API. The "loader" is defined as any context that is tied to a single stream. + * @throws IOException if thrown during reading. + */ + abstract void fullyReadElementFromBuffer(SideEffectConsumer consumer) throws IOException; + + /** + * Initialize the element loader and perform a fully-materialized deep read of the data from a file using + * IonElement API. The "loader" is defined as any context that is tied to a single stream. + * @throws IOException if thrown during reading. + */ + abstract void fullyReadElementFromFile(SideEffectConsumer consumer) throws IOException; + @Override public void setUpTrial() throws IOException { inputFile = options.convertFileIfNecessary(originalFile).toFile(); @@ -124,6 +138,12 @@ public final Task getTask() { } else { return this::fullyReadDomFromFile; } + } else if (options.api == API.ION_ELEMENT_DOM) { + if (buffer != null) { + return this::fullyReadElementFromBuffer; + } else { + return this::fullyReadElementFromFile; + } } else { throw new IllegalStateException("Illegal combination of options."); } diff --git a/src/com/amazon/ion/benchmark/MeasurableWriteTask.java b/src/com/amazon/ion/benchmark/MeasurableWriteTask.java index 496c9fa..51f316b 100644 --- a/src/com/amazon/ion/benchmark/MeasurableWriteTask.java +++ b/src/com/amazon/ion/benchmark/MeasurableWriteTask.java @@ -72,6 +72,14 @@ interface WriteInstruction { */ abstract void generateWriteInstructionsStreaming(Consumer> instructionsSink) throws IOException; + /** + * Generate a sequence of WriteInstructions that re-write the input file with the configured options using the + * IonElement API. + * @param instructionsSink the sink for the sequence of generated WriteInstructions. + * @throws IOException if thrown when generating WriteInstructions. + */ + abstract void generateWriteInstructionsElement(Consumer> instructionsSink) throws IOException; + /** * @return a new writer context instance. * @param outputStream the OutputStream to which the new writer will write. @@ -96,6 +104,9 @@ public void setUpTrial() throws IOException { case DOM: generateWriteInstructionsDom(writeInstructions::add); break; + case ION_ELEMENT_DOM: + generateWriteInstructionsElement(writeInstructions::add); + break; } } diff --git a/src/com/amazon/ion/benchmark/OptionsCombinationBase.java b/src/com/amazon/ion/benchmark/OptionsCombinationBase.java index 32be898..6c94d39 100644 --- a/src/com/amazon/ion/benchmark/OptionsCombinationBase.java +++ b/src/com/amazon/ion/benchmark/OptionsCombinationBase.java @@ -92,6 +92,11 @@ static T getOrDefault(IonStruct options, String fieldName, Function ((IonInt) val).intValue(), Integer.MAX_VALUE); jsonUseBigDecimals = getOrDefault(optionsCombinationStruct, JSON_USE_BIG_DECIMALS_NAME, val -> ((IonBool) val).booleanValue(), true); autoFlush = getOrDefault(optionsCombinationStruct, AUTO_FLUSH_ENABLED, val -> ((IonBool) val).booleanValue(), false); + + // Validate that ION_ELEMENT_DOM is only used with Ion formats + if (api == API.ION_ELEMENT_DOM && !format.isIon()) { + throw new IllegalArgumentException("ION_ELEMENT_DOM API can only be used with Ion formats (ion_binary or ion_text), not with " + format.name().toLowerCase()); + } } /** diff --git a/src/com/amazon/ion/benchmark/OptionsMatrixBase.java b/src/com/amazon/ion/benchmark/OptionsMatrixBase.java index b971002..f1e062c 100644 --- a/src/com/amazon/ion/benchmark/OptionsMatrixBase.java +++ b/src/com/amazon/ion/benchmark/OptionsMatrixBase.java @@ -54,6 +54,9 @@ abstract class OptionsMatrixBase { static final Predicate OPTION_ONLY_APPLIES_TO_ION_STREAMING = s -> { return OPTION_ONLY_APPLIES_TO_ION.test(s) && API.STREAMING.name().equals(getStringValue(s, API_NAME)); }; + static final Predicate OPTION_ONLY_APPLIES_TO_ION_ELEMENT_DOM = s -> { + return OPTION_ONLY_APPLIES_TO_ION.test(s) && API.ION_ELEMENT_DOM.name().equals(getStringValue(s, API_NAME)); + }; static final Predicate OPTION_ONLY_APPLIES_TO_JSON = s -> { return Format.JSON.name().equals(getStringValue(s, FORMAT_NAME)); }; diff --git a/tst/com/amazon/ion/benchmark/OptionsTest.java b/tst/com/amazon/ion/benchmark/OptionsTest.java index 2500951..30df465 100644 --- a/tst/com/amazon/ion/benchmark/OptionsTest.java +++ b/tst/com/amazon/ion/benchmark/OptionsTest.java @@ -660,6 +660,46 @@ public void writeTextUsingDom() throws Exception { assertTrue(expectedCombinations.isEmpty()); } + @Test + public void writeTextUsingIonElementDom() throws Exception { + List optionsCombinations = parseOptionsCombinations( + "write", + "--format", + "ion_text", + "--api", + "ion_element_dom", + "--io-type", + "buffer", + "--io-type", + "file", + "textStructs.ion" + ); + assertEquals(2, optionsCombinations.size()); + List expectedCombinations = new ArrayList<>(2); + expectedCombinations.add(ExpectedWriteOptionsCombination.defaultOptions() + .api(API.ION_ELEMENT_DOM) + .format(Format.ION_TEXT) + .ioType(IoType.BUFFER) + ); + expectedCombinations.add(ExpectedWriteOptionsCombination.defaultOptions() + .api(API.ION_ELEMENT_DOM) + .format(Format.ION_TEXT) + .ioType(IoType.FILE) + ); + + for (WriteOptionsCombination optionsCombination : optionsCombinations) { + expectedCombinations.removeIf(candidate -> { + return candidate.api == API.ION_ELEMENT_DOM + && candidate.format == Format.ION_TEXT + && candidate.ioType == optionsCombination.ioType; + }); + + assertWriteTaskExecutesCorrectly("binaryStructs.10n", optionsCombination, Format.ION_TEXT, optionsCombination.ioType); + assertWriteTaskExecutesCorrectly("textStructs.ion", optionsCombination, Format.ION_TEXT, optionsCombination.ioType); + } + assertTrue(expectedCombinations.isEmpty()); + } + @Test public void readBothTextAndIonUsingBothDomAndReader() throws Exception { List optionsCombinations = parseOptionsCombinations( @@ -718,6 +758,51 @@ public void readBothTextAndIonUsingBothDomAndReader() throws Exception { assertTrue(expectedCombinations.isEmpty()); } + @Test + public void readBothTextAndBinaryUsingIonElementDom() throws Exception { + List optionsCombinations = parseOptionsCombinations( + "read", + "--format", + "ion_text", + "--format", + "ion_binary", + "--io-type", + "buffer", + "--api", + "ion_element_dom", + "binaryStructs.10n" + ); + assertEquals(2, optionsCombinations.size()); + List expectedCombinations = new ArrayList<>(2); + expectedCombinations.add(ExpectedReadOptionsCombination.defaultOptions() + .ioType(IoType.BUFFER) + .format(Format.ION_TEXT) + .api(API.ION_ELEMENT_DOM) + ); + expectedCombinations.add(ExpectedReadOptionsCombination.defaultOptions() + .ioType(IoType.BUFFER) + .format(Format.ION_BINARY) + .api(API.ION_ELEMENT_DOM) + ); + for (ReadOptionsCombination optionsCombination : optionsCombinations) { + expectedCombinations.removeIf(candidate -> candidate.format == optionsCombination.format + && candidate.api == optionsCombination.api); + assertReadTaskExecutesCorrectly( + "binaryStructs.10n", + optionsCombination, + optionsCombination.format, + optionsCombination.format != Format.ION_BINARY + ); + assertReadTaskExecutesCorrectly( + "textStructs.ion", + optionsCombination, + optionsCombination.format, + optionsCombination.format != Format.ION_TEXT + ); + } + assertTrue(expectedCombinations.isEmpty()); + } + @Test public void readBinaryWithLimit() throws Exception { List optionsCombinations = parseOptionsCombinations( @@ -782,6 +867,37 @@ public void readBinaryWithLimitFromFileUsingDom() throws Exception { assertTrue(expectedCombinations.isEmpty()); } + @Test + public void readBinaryAndTextWithLimitFromFileUsingIonElementDom() throws Exception { + List optionsCombinations = parseOptionsCombinations( + "read", + "--limit", + "1", + "--io-type", + "file", + "--format", + "ion_text", + "--format", + "ion_binary", + "--api", + "ion_element_dom", + "binaryStructs.10n" + ); + assertEquals(2, optionsCombinations.size()); + List expectedCombinations = new ArrayList<>(2); + expectedCombinations.add(ExpectedReadOptionsCombination.defaultOptions().limit(1).api(API.ION_ELEMENT_DOM).format(Format.ION_BINARY)); + expectedCombinations.add(ExpectedReadOptionsCombination.defaultOptions().limit(1).api(API.ION_ELEMENT_DOM).format(Format.ION_TEXT)); + + for (ReadOptionsCombination optionsCombination : optionsCombinations) { + expectedCombinations.removeIf(candidate -> candidate.format == optionsCombination.format); + assertEquals(1, optionsCombination.limit); + + assertReadTaskExecutesCorrectly("binaryStructs.10n", optionsCombination, optionsCombination.format, true); + assertReadTaskExecutesCorrectly("textStructs.ion", optionsCombination, optionsCombination.format, true); + } + assertTrue(expectedCombinations.isEmpty()); + } + @Test public void writeBinaryWithLimitUsingWriterAndDOM() throws Exception { List optionsCombinations = parseOptionsCombinations( @@ -809,6 +925,30 @@ public void writeBinaryWithLimitUsingWriterAndDOM() throws Exception { assertTrue(expectedCombinations.isEmpty()); } + @Test + public void writeBinaryWithLimitUsingIonElementDOM() throws Exception { + List optionsCombinations = parseOptionsCombinations( + "write", + "--limit", + "1", + "--api", + "ion_element_dom", + "binaryStructs.10n" + ); + assertEquals(1, optionsCombinations.size()); + List expectedCombinations = new ArrayList<>(1); + expectedCombinations.add(ExpectedWriteOptionsCombination.defaultOptions().limit(1).api(API.ION_ELEMENT_DOM)); + + for (WriteOptionsCombination optionsCombination : optionsCombinations) { + expectedCombinations.removeIf(candidate -> candidate.api == optionsCombination.api); + assertEquals(1, optionsCombination.limit); + + assertWriteTaskExecutesCorrectly("binaryStructs.10n", optionsCombination, Format.ION_BINARY, IoType.FILE); + assertWriteTaskExecutesCorrectly("textStructs.ion", optionsCombination, Format.ION_BINARY, IoType.FILE); + } + assertTrue(expectedCombinations.isEmpty()); + } + @Test public void profileWithMultipleCombinationsRaisesError() { assertThrows(