diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java index fefde8900..02aee4318 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java @@ -6,6 +6,7 @@ import fr.insee.vtl.engine.VtlScriptEngine; import fr.insee.vtl.engine.exceptions.InvalidArgumentException; +import fr.insee.vtl.engine.exceptions.UndefinedVariableException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.*; @@ -13,6 +14,7 @@ import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.misc.Interval; @@ -106,16 +108,59 @@ private static AggregationExpression convertToAggregation( @Override public DatasetExpression visitKeepOrDropClause(VtlParser.KeepOrDropClauseContext ctx) { - // Normalize to keep operation. - var keep = ctx.op.getType() == VtlParser.KEEP; - var names = ctx.componentID().stream().map(ClauseVisitor::getName).collect(Collectors.toSet()); - List columnNames = - datasetExpression.getDataStructure().values().stream() - .map(Dataset.Component::getName) - .filter(name -> keep == names.contains(name)) - .collect(Collectors.toList()); - return processingEngine.executeProject(datasetExpression, columnNames); + // The type of the op can either be KEEP or DROP. + boolean keep = ctx.op.getType() == VtlParser.KEEP; + + // Dataset identifiers (role = IDENTIFIER) + Map identifiers = + datasetExpression.getDataStructure().getIdentifiers().stream() + .collect(Collectors.toMap(Structured.Component::getName, Function.identity())); + + var columns = + ctx.componentID().stream() + .collect(Collectors.toMap(ClauseVisitor::getName, Function.identity())); + + var structure = datasetExpression.getDataStructure(); + + // Evaluate that all requested columns must exist in the dataset or raise an error + // TODO: Is that no handled already? + for (String col : columns.keySet()) { + if (!structure.containsKey(col)) { + throw new VtlRuntimeException( + new UndefinedVariableException(col, fromContext(columns.get(col)))); + } + } + + // VTL specification: identifiers must not appear explicitly in KEEP + // TODO: Use multi errors that noah created? + for (String col : columns.keySet()) { + if (structure.get(col).isIdentifier()) { + throw new VtlRuntimeException( + new InvalidArgumentException( + "cannot keep/drop identifiers", fromContext(columns.get(col)))); + } + } + + // Build result set: + // + KEEP: identifiers + requested columns + // + DROP: (all columns - requested) + identifiers + final Set resultSet = new LinkedHashSet<>(); + resultSet.addAll(identifiers.keySet()); + if (keep) { + resultSet.addAll(columns.keySet()); + } else { + for (String col : structure.keySet()) { + if (!columns.keySet().contains(col)) { + resultSet.add(col); + } + } + } + + // Retrieve the output column names (identifiers + requested) + final List outputColumns = + structure.keySet().stream().filter(resultSet::contains).collect(Collectors.toList()); + return processingEngine.executeProject(datasetExpression, outputColumns); } @Override diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java index d927d21d8..e0cb8a94c 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java @@ -33,24 +33,37 @@ public static Condition atPosition( public static Condition atPosition( Integer startLine, Integer endLine, Integer startColumn, Integer endColumn) { - return new Condition<>( - throwable -> { - var scriptException = (VtlScriptException) throwable; - var position = scriptException.getPosition(); - return position.startLine().equals(startLine) - && position.endLine().equals(endLine) - && position.startColumn().equals(startColumn) - && position.endColumn().equals(endColumn); - }, - "at position <%d:%d-%d:%d>", - startLine, - endLine, - startColumn, - endColumn); - } - - public static > Boolean test(T left, T right) { - return true; + return new Condition() { + @Override + public boolean matches(T throwable) { + if (!(throwable instanceof VtlScriptException scriptException)) { + return false; + } + var position = scriptException.getPosition(); + boolean matches = + position.startLine().equals(startLine) + && position.endLine().equals(endLine) + && position.startColumn().equals(startColumn) + && position.endColumn().equals(endColumn); + + // Set description that includes actual position if it doesn't match + if (matches) { + describedAs("at position <%d:%d-%d:%d>", startLine, startColumn, endLine, endColumn); + } else { + describedAs( + "at position <%d:%d-%d:%d> but was <%d:%d-%d:%d>", + startLine, + startColumn, + endLine, + endColumn, + position.startLine(), + position.startColumn(), + position.endLine(), + position.endColumn()); + } + return matches; + } + }; } @BeforeEach diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/utils/dag/DagTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/utils/dag/DagTest.java index 841ba57e8..fcb4a6037 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/utils/dag/DagTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/utils/dag/DagTest.java @@ -199,11 +199,11 @@ void testDagSimpleExampleWithReordering() throws ScriptException { void testDagCycle() { final String script = """ - e := a; - b := a; - c := b; - a := c; - f := a;"""; + e := a; + b := a; + c := b; + a := c; + f := a;"""; final Positioned.Position mainPosition = getPositionOfStatementInScript("a := c", script); final List otherPositions = @@ -231,14 +231,14 @@ void testDagCycle() { void testMultipleCycles() { final String script = """ - h := g; - i := join(h, input_ds); - g := i; - e := a; - b := a; - c := b; - a := c; - f := a;"""; + h := g; + i := join(h, input_ds); + g := i; + e := a; + b := a; + c := b; + a := c; + f := a;"""; final Positioned.Position mainExceptionMainPosition = getPositionOfStatementInScript("g := i", script); @@ -446,8 +446,8 @@ void testDagIfExpr() throws ScriptException { engine.getContext().setAttribute("ds_2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval( "res := if ds1 > ds2 then ds1 else ds2; " - + "ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " - + "ds2 := ds_2[keep id, long1][rename long1 to bool_var];"); + + "ds1 := ds_1[keep long1][rename long1 to bool_var]; " + + "ds2 := ds_2[keep long1][rename long1 to bool_var];"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( @@ -463,7 +463,7 @@ void testDagCaseExpr() throws ScriptException { engine.getContext().setAttribute("ds_2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval( "res0 <- tmp0[calc c := case when long1 > 30 then \"ok\" else \"ko\"][drop long1]; " - + "tmp0 := ds_1[keep id, long1];"); + + "tmp0 := ds_1[keep long1];"); Object res0 = engine.getContext().getAttribute("res0"); assertThat(((Dataset) res0).getDataAsMap()) .containsExactlyInAnyOrder( @@ -474,7 +474,7 @@ void testDagCaseExpr() throws ScriptException { assertThat(((Dataset) res0).getDataStructure().get("c").getType()).isEqualTo(String.class); engine.eval( "res1 <- tmp1[calc c := case when long1 > 30 then 1 else 0][drop long1]; " - + "tmp1 := ds_1[keep id, long1];"); + + "tmp1 := ds_1[keep long1];"); Object res1 = engine.getContext().getAttribute("res1"); assertThat(((Dataset) res1).getDataAsMap()) .containsExactlyInAnyOrder( @@ -484,9 +484,9 @@ void testDagCaseExpr() throws ScriptException { Map.of("id", "Franck", "c", 1L)); assertThat(((Dataset) res1).getDataStructure().get("c").getType()).isEqualTo(Long.class); engine.eval( - "tmp2_alt_ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " + "tmp2_alt_ds1 := ds_1[keep long1][rename long1 to bool_var]; " + "res2 <- case when tmp2_alt_ds1 < 30 then tmp2_alt_ds1 else tmp2_alt_ds2; " - + "tmp2_alt_ds2 := ds_2[keep id, long1][rename long1 to bool_var];"); + + "tmp2_alt_ds2 := ds_2[keep long1][rename long1 to bool_var];"); Object resDs = engine.getContext().getAttribute("res2"); assertThat(((Dataset) resDs).getDataAsMap()) .containsExactlyInAnyOrder( @@ -498,7 +498,7 @@ void testDagCaseExpr() throws ScriptException { @Test void testDagNvlExpr() throws ScriptException { engine.getContext().setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - engine.eval("res <- nvl(tmp1[keep id, long1], 0); tmp1 := ds1;"); + engine.eval("res <- nvl(tmp1[keep long1], 0); tmp1 := ds1;"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( @@ -512,7 +512,7 @@ void testDagNvlExpr() throws ScriptException { @Test void testDagNvlImplicitCast() throws ScriptException { engine.getContext().setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - engine.eval("res := nvl(tmp1[keep id, long1], 0.1); tmp1 <- ds1;"); + engine.eval("res := nvl(tmp1[keep long1], 0.1); tmp1 <- ds1;"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( @@ -528,7 +528,7 @@ void testDagUnaryExpr() throws ScriptException { ScriptContext context = engine.getContext(); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := + tmp1[keep id, long1, double1]; tmp1 <- ds2;"); + Object res = engine.eval("res := + tmp1[keep long1, double1]; tmp1 <- ds2;"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 150L, "double1", 1.1D), @@ -537,7 +537,7 @@ void testDagUnaryExpr() throws ScriptException { assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Long.class); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res2 = engine.eval("res2 := - tmp2[keep id, long1, double1]; tmp2 := ds2;"); + Object res2 = engine.eval("res2 := - tmp2[keep long1, double1]; tmp2 := ds2;"); assertThat(((Dataset) res2).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", -150L, "double1", -1.1D), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java index 5c25d981b..992e56f19 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java @@ -87,7 +87,7 @@ public void testManyCalc() throws ScriptException { } @Test - public void testCalcRoleModifier() throws ScriptException { + public void testCalcRoleModifier_measuresAndAttributesOk() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( List.of( @@ -194,7 +194,8 @@ public void testKeepDropClause() throws ScriptException { ScriptContext context = engine.getContext(); context.setAttribute("ds1", dataset, ScriptContext.ENGINE_SCOPE); - engine.eval("ds2 := ds1[keep name, age];"); + // KEEP: identifiers must not be listed explicitly; they are implicitly preserved. + engine.eval("ds2 := ds1[keep age];"); assertThat(engine.getContext().getAttribute("ds2")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("ds2")).getDataAsMap()) @@ -213,6 +214,27 @@ public void testKeepDropClause() throws ScriptException { Map.of("name", "Franck", "age", 12L)); } + /** KEEP/DROP: listing identifiers explicitly must raise a script error. */ + @Test + public void testKeepDropClause_identifierExplicitShouldFail() { + InMemoryDataset dataset = + new InMemoryDataset( + List.of( + Map.of("name", "Hadrien", "age", 10L, "weight", 11L), + Map.of("name", "Nico", "age", 11L, "weight", 10L), + Map.of("name", "Franck", "age", 12L, "weight", 9L)), + Map.of("name", String.class, "age", Long.class, "weight", Long.class), + Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE)); + + ScriptContext context = engine.getContext(); + context.setAttribute("ds1", dataset, ScriptContext.ENGINE_SCOPE); + + assertThatThrownBy(() -> engine.eval("ds := ds1[keep name, age];")) + .isInstanceOf(VtlScriptException.class) + .hasMessage("cannot keep/drop identifiers") + .is(atPosition(0, 15, 19)); + } + @Test public void testAggregateType() { InMemoryDataset dataset = diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java index 3d61e46ba..d3cb18544 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java @@ -52,7 +52,7 @@ public void testPlus() throws ScriptException { assertThat(context.getAttribute("plus2")).isEqualTo(5.0); context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := ds1[keep id, long1, long2] + ds1[keep id, long1, long2];"); + Object res = engine.eval("res := ds1[keep long1, long2] + ds1[keep long1, long2];"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 60L, "long2", 600L), @@ -75,7 +75,7 @@ public void testMinus() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := ds2[keep id, long1] - ds1[keep id, long1] + 1;"); + Object res = engine.eval("res := ds2[keep long1] - ds1[keep long1] + 1;"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 141L), @@ -92,7 +92,7 @@ public void testConcat() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := ds2[keep id, string1] || \" \" || ds1[keep id, string1];"); + Object res = engine.eval("res := ds2[keep string1] || \" \" || ds1[keep string1];"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "string1", "hadrien hadrien"), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java index c5822ea0f..353838880 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java @@ -64,8 +64,7 @@ public void testArithmeticExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = - engine.eval("res := round(ds1[keep id, long1, double1] * ds2[keep id, long1, double1]);"); + Object res = engine.eval("res := round(ds1[keep long1, double1] * ds2[keep long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 1500.0, "double1", 1.0), @@ -83,8 +82,7 @@ public void testArithmeticExpr() throws ScriptException { engine.eval("div4 := 3.0 / 1.5;"); assertThat(context.getAttribute("div4")).isEqualTo(2.0); - res = - engine.eval("res2 := round(ds1[keep id, long1, double1] / ds2[keep id, long1, double1]);"); + res = engine.eval("res2 := round(ds1[keep long1, double1] / ds2[keep long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 0.0, "double1", 1.0), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java index 7e7216f6d..d9ba19e7d 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java @@ -100,8 +100,8 @@ public void testOnDatasets() throws ScriptException { context.setAttribute("ds_1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds_2", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.eval( - "ds1 := ds_1[keep id, bool2][rename bool2 to bool1]; " - + "ds2 := ds_2[keep id, bool1]; " + "ds1 := ds_1[keep bool2][rename bool2 to bool1]; " + + "ds2 := ds_2[keep bool1]; " + "andDs := ds1 and ds2; " + "orDs := ds1 or ds2; " + "xorDs := ds1 xor ds2; "); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java index 70921ac91..297becc48 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java @@ -73,7 +73,7 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - engine.eval("equal := ds1[keep id, long1] = ds2[keep id, long1];"); + engine.eval("equal := ds1[keep long1] = ds2[keep long1];"); var equal = engine.getContext().getAttribute("equal"); assertThat(((Dataset) equal).getDataAsMap()) .containsExactlyInAnyOrder( @@ -90,7 +90,7 @@ public void testComparisonExpr() throws ScriptException { assertThat((Boolean) context.getAttribute("long1")).isTrue(); engine.eval("mix1 := 6 <> (3*20.0);"); assertThat((Boolean) context.getAttribute("mix1")).isTrue(); - engine.eval("notEqual := ds1[keep id, long1] <> ds2[keep id, long1];"); + engine.eval("notEqual := ds1[keep long1] <> ds2[keep long1];"); var notEqual = engine.getContext().getAttribute("notEqual"); assertThat(((Dataset) notEqual).getDataAsMap()) .containsExactlyInAnyOrder( @@ -106,7 +106,7 @@ public void testComparisonExpr() throws ScriptException { assertThat((Boolean) context.getAttribute("lt1")).isFalse(); engine.eval("mix2 := 6 < 6.1;"); assertThat((Boolean) context.getAttribute("mix2")).isTrue(); - engine.eval("lt2 := ds1[keep id, long1] < ds2[keep id, long1];"); + engine.eval("lt2 := ds1[keep long1] < ds2[keep long1];"); var lt = engine.getContext().getAttribute("lt2"); assertThat(((Dataset) lt).getDataAsMap()) .containsExactlyInAnyOrder( @@ -122,7 +122,7 @@ public void testComparisonExpr() throws ScriptException { assertThat((Boolean) context.getAttribute("mt1")).isTrue(); engine.eval("mix4 := 6 > 6.1;"); assertThat((Boolean) context.getAttribute("mix4")).isFalse(); - engine.eval("mt2 := ds1[keep id, long1] > ds2[keep id, long1];"); + engine.eval("mt2 := ds1[keep long1] > ds2[keep long1];"); var mt = engine.getContext().getAttribute("mt2"); assertThat(((Dataset) mt).getDataAsMap()) .containsExactlyInAnyOrder( @@ -139,7 +139,7 @@ public void testComparisonExpr() throws ScriptException { engine.eval("mix5 := 6 <= 6.1;"); assertThat((Boolean) context.getAttribute("mix5")).isTrue(); - engine.eval("le2 := ds1[keep id, long1] <= ds2[keep id, long1];"); + engine.eval("le2 := ds1[keep long1] <= ds2[keep long1];"); var le = engine.getContext().getAttribute("le2"); assertThat(((Dataset) le).getDataAsMap()) .containsExactlyInAnyOrder( @@ -156,7 +156,7 @@ public void testComparisonExpr() throws ScriptException { engine.eval("mix6 := 6 >= 6.1;"); assertThat((Boolean) context.getAttribute("mix6")).isFalse(); - engine.eval("me2 := ds1[keep id, long1] >= ds2[keep id, long1];"); + engine.eval("me2 := ds1[keep long1] >= ds2[keep long1];"); var me = engine.getContext().getAttribute("me2"); assertThat(((Dataset) me).getDataAsMap()) .containsExactlyInAnyOrder( @@ -198,7 +198,7 @@ public void testInNotIn() throws ScriptException { assertThat((Boolean) engine.getContext().getAttribute("res4")).isTrue(); engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - engine.eval("me := ds[keep id, long1, string1] in {\"toto\", \"franck\"};"); + engine.eval("me := ds[keep long1, string1] in {\"toto\", \"franck\"};"); var in = engine.getContext().getAttribute("me"); assertThat(((Dataset) in).getDataAsMap()) .containsExactlyInAnyOrder( diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java index 2994c97c7..5ce05c628 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java @@ -46,8 +46,8 @@ public void testIfExpr() throws ScriptException { engine.getContext().setAttribute("ds_1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.getContext().setAttribute("ds_2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval( - "ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " - + "ds2 := ds_2[keep id, long1][rename long1 to bool_var]; " + "ds1 := ds_1[keep long1][rename long1 to bool_var]; " + + "ds2 := ds_2[keep long1][rename long1 to bool_var]; " + "res := if ds1 > ds2 then ds1 else ds2;"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) @@ -73,7 +73,7 @@ public void testCaseExpr() throws ScriptException { engine.getContext().setAttribute("ds_1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.getContext().setAttribute("ds_2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval( - "ds1 := ds_1[keep id, long1]; " + "ds1 := ds_1[keep long1]; " + "res <- ds1[calc c := case when long1 > 30 then \"ok\" else \"ko\"][drop long1];"); Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) @@ -84,7 +84,7 @@ public void testCaseExpr() throws ScriptException { Map.of("id", "Franck", "c", "ok")); assertThat(((Dataset) res).getDataStructure().get("c").getType()).isEqualTo(String.class); engine.eval( - "ds2 := ds_1[keep id, long1]; " + "ds2 := ds_1[keep long1]; " + "res2 <- ds2[calc c := case when long1 > 30 then 1 else 0][drop long1];"); Object res2 = engine.getContext().getAttribute("res2"); assertThat(((Dataset) res2).getDataAsMap()) @@ -95,8 +95,8 @@ public void testCaseExpr() throws ScriptException { Map.of("id", "Franck", "c", 1L)); assertThat(((Dataset) res2).getDataStructure().get("c").getType()).isEqualTo(Long.class); engine.eval( - "ds3 := ds_1[keep id, long1][rename long1 to bool_var];" - + "ds4 := ds_2[keep id, long1][rename long1 to bool_var]; " + "ds3 := ds_1[keep long1][rename long1 to bool_var];" + + "ds4 := ds_2[keep long1][rename long1 to bool_var]; " + "res_ds <- case when ds3 < 30 then ds3 else ds4;"); Object res_ds = engine.getContext().getAttribute("res_ds"); assertThat(((Dataset) res_ds).getDataAsMap()) @@ -115,7 +115,7 @@ public void testNvlExpr() throws ScriptException { assertThat(context.getAttribute("s2")).isEqualTo("default"); engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - engine.eval("res := nvl(ds[keep id, long1], 0);"); + engine.eval("res := nvl(ds[keep long1], 0);"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( @@ -142,7 +142,7 @@ public void testNvlImplicitCast() throws ScriptException { assertThat(context.getAttribute("s2")).isEqualTo(1.1D); engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - engine.eval("res := nvl(ds[keep id, long1], 0.1);"); + engine.eval("res := nvl(ds[keep long1], 0.1);"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java index 45ef35973..bbe9aea24 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java @@ -44,7 +44,7 @@ public void testUnaryExpr() throws ScriptException { assertThat(context.getAttribute("plus1")).isEqualTo(1.5D); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res1 := + ds2[keep id, long1, double1];"); + Object res = engine.eval("res1 := + ds2[keep long1, double1];"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 150L, "double1", 1.1D), @@ -57,7 +57,7 @@ public void testUnaryExpr() throws ScriptException { engine.eval("plus3 := - 1.5;"); assertThat(context.getAttribute("plus3")).isEqualTo(-1.5D); - res = engine.eval("res2 := - ds2[keep id, long1, double1];"); + res = engine.eval("res2 := - ds2[keep long1, double1];"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", -150L, "double1", -1.1D), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java index 9e307d085..e19a3d273 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java @@ -50,7 +50,7 @@ public void testBetweenAtom() throws ScriptException { assertThat((Boolean) context.getAttribute("b2")).isFalse(); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := between(ds[keep id, long1, double2], 5, 15);"); + Object res = engine.eval("res := between(ds[keep long1, double2], 5, 15);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", false, "double2", false), @@ -80,8 +80,7 @@ public void testCharsetMatchAtom() throws ScriptException { assertThat((Boolean) context.getAttribute("t3")).isFalse(); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = - engine.eval("res := match_characters(ds[keep id, string1, string2], \"(.*)o(.*)\");"); + Object res = engine.eval("res := match_characters(ds[keep string1, string2], \"(.*)o(.*)\");"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "string1", true, "string2", false), @@ -114,7 +113,7 @@ public void testIsNullAtom() throws ScriptException { assertThat((Boolean) context.getAttribute("n1")).isFalse(); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := isnull(ds[keep id, string1, bool1]);"); + Object res = engine.eval("res := isnull(ds[keep string1, bool1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "string1", false, "bool1", false), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java index f83d6837f..787e230b0 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java @@ -43,7 +43,7 @@ public void testLevenshteinAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval( - "res := levenshtein(ds[keep id, string1], ds[keep id, string2][rename string2 to string1])[rename string1 to lev];"); + "res := levenshtein(ds[keep string1], ds[keep string2][rename string2 to string1])[rename string1 to lev];"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "lev", 3L), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java index e79293263..8042158e0 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java @@ -230,9 +230,8 @@ public void testLeftJoinMixedStructure() throws ScriptException { engine.getContext().setAttribute("ds1", dataset1, ScriptContext.ENGINE_SCOPE); engine.getContext().setAttribute("ds2", dataset2, ScriptContext.ENGINE_SCOPE); - engine.eval( - "unionData := union(ds1[keep id, measure1, measure2], ds2[keep id, measure1, measure2]);"); - engine.eval("ds1_keep := ds1[keep id, color];"); + engine.eval("unionData := union(ds1[keep measure1, measure2], ds2[keep measure1, measure2]);"); + engine.eval("ds1_keep := ds1[keep color];"); engine.eval("joinData := left_join(unionData, ds1_keep);"); Dataset joinData = (Dataset) engine.getBindings(ScriptContext.ENGINE_SCOPE).get("joinData"); @@ -260,7 +259,7 @@ public void testInnerJoin() throws ScriptException { engine.getContext().setAttribute("ds_1", ds1, ScriptContext.ENGINE_SCOPE); engine.getContext().setAttribute("ds_2", ds2, ScriptContext.ENGINE_SCOPE); - engine.eval("result1 := inner_join(ds_1[keep id1, id2, m1] as ds1, ds_2 as ds2);"); + engine.eval("result1 := inner_join(ds_1[keep m1] as ds1, ds_2 as ds2);"); var result = (Dataset) engine.getContext().getAttribute("result1"); assertThat(result.getColumnNames()).containsExactlyInAnyOrder("id1", "id2", "m1", "m2"); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java index d26b9617a..558e1f720 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java @@ -85,7 +85,7 @@ public void testCeil() throws ScriptException { assertThat(context.getAttribute("d")).isEqualTo(0L); context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := ceil(ds[keep id, long1, double1]);"); + Object res = engine.eval("res := ceil(ds[keep long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "long1", 150L, "double1", 2L), @@ -112,7 +112,7 @@ public void testFloor() throws ScriptException { assertThat(context.getAttribute("d")).isEqualTo(-1L); context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := floor(ds[keep id, double1]);"); + Object res = engine.eval("res := floor(ds[keep double1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "double1", 1L), @@ -135,7 +135,7 @@ public void testAbs() throws ScriptException { assertThat(context.getAttribute("b")).isEqualTo(5.5D); context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := abs(ds[keep id, double1]);"); + Object res = engine.eval("res := abs(ds[keep double1]);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "double1", 1.1D), @@ -164,7 +164,7 @@ public void testExp() throws ScriptException { assertThat(((Double) context.getAttribute("d"))) .isCloseTo(0.367D, Percentage.withPercentage(1)); context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := floor(exp(ds[keep id, double1, long1]));"); + Object res = engine.eval("res := floor(exp(ds[keep double1, long1]));"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "double1", 3L, "long1", 9223372036854775807L), @@ -192,7 +192,7 @@ public void testLn() throws ScriptException { .isCloseTo(-0.69D, Percentage.withPercentage(1)); context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := floor(ln(abs(ds[keep id, double1])));"); + Object res = engine.eval("res := floor(ln(abs(ds[keep double1])));"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Hadrien", "double1", 0L), @@ -221,7 +221,7 @@ public void testRound() throws ScriptException { assertThat(context.getAttribute("e")).isEqualTo(12350D); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := round(ds[keep id, long1, double2], 1);"); + Object res = engine.eval("res := round(ds[keep long1, double2], 1);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 30.0D, "double2", 1.2D), @@ -258,7 +258,7 @@ public void testTrunc() throws ScriptException { assertThat(context.getAttribute("e")).isEqualTo(12340D); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := trunc(ds[keep id, long1, double2], 1);"); + Object res = engine.eval("res := trunc(ds[keep long1, double2], 1);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 30.0D, "double2", 1.2D), @@ -289,7 +289,7 @@ public void testSqrt() throws ScriptException { assertThat(context.getAttribute("c")).isEqualTo(0D); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := round(sqrt(ds[keep id, long1, double2]));"); + Object res = engine.eval("res := round(sqrt(ds[keep long1, double2]));"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 5.0D, "double2", 1D), @@ -321,7 +321,7 @@ public void testMod() throws ScriptException { assertThat(context.getAttribute("d")).isEqualTo(9D); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := trunc(mod(ds[keep id, long1, double2], 2), 1);"); + Object res = engine.eval("res := trunc(mod(ds[keep long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 0.0D, "double2", 1.2D), @@ -353,7 +353,7 @@ public void testPower() throws ScriptException { assertThat(context.getAttribute("e")).isEqualTo(-125D); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := trunc(power(ds[keep id, long1, double2], 2), 1);"); + Object res = engine.eval("res := trunc(power(ds[keep long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 900.0D, "double2", 1.4D), @@ -383,7 +383,7 @@ public void testRandom() throws ScriptException { assertThat((Double) context.getAttribute("a")).isCloseTo(0.72D, Percentage.withPercentage(0.2)); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := trunc(random(ds[keep id, long1], 200), 1);"); + Object res = engine.eval("res := trunc(random(ds[keep long1], 200), 1);"); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); assertThatThrownBy( @@ -410,7 +410,7 @@ public void testLog() throws ScriptException { .isCloseTo(3.01D, Percentage.withPercentage(0.01)); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := trunc(log(ds[keep id, long1, double2], 2), 1);"); + Object res = engine.eval("res := trunc(log(ds[keep long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "long1", 4.9D, "double2", 0.2D), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java index d548831ba..510b58d6d 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java @@ -265,9 +265,9 @@ public void testUnion456Issue() throws ScriptException { context.getBindings(ScriptContext.ENGINE_SCOPE).put("MULTIMODE", multimodeDs); engine.eval( - "TEMP_RACINE := MULTIMODE [keep interrogationId, FOO];\n" + "TEMP_RACINE := MULTIMODE [keep FOO];\n" + "RACINE := union(TEMP_RACINE, TEMP_RACINE) ;\n" - + "TEMP_LOOP := MULTIMODE [keep interrogationId, LOOP, LOOP.FOO1]\n" + + "TEMP_LOOP := MULTIMODE [keep LOOP.FOO1]\n" + " [filter LOOP <> \"\"]\n" + " [rename LOOP.FOO1 to FOO1];\n" + "LOOP <- union(TEMP_LOOP, TEMP_LOOP);"); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java index d291258b9..516a48c45 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java @@ -86,12 +86,12 @@ public void testUnaryStringFunction() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.eval( - "dsTrim := trim(ds[keep id, string1]); " - + "dsLTrim := ltrim(ds[keep id, string1]); " - + "dsRTrim := rtrim(ds[keep id, string1]); " - + "dsUpper := upper(ds[keep id, string1]); " - + "dsLower := lower(ds[keep id, string1]); " - + "dsLen := length(ds[keep id, string1]);"); + "dsTrim := trim(ds[keep string1]); " + + "dsLTrim := ltrim(ds[keep string1]); " + + "dsRTrim := rtrim(ds[keep string1]); " + + "dsUpper := upper(ds[keep string1]); " + + "dsLower := lower(ds[keep string1]); " + + "dsLen := length(ds[keep string1]);"); assertThat(((Dataset) context.getAttribute("dsTrim")).getDataAsMap().get(0)) .isEqualTo(Map.of("id", "Toto", "string1", "toto")); assertThat(((Dataset) context.getAttribute("dsLTrim")).getDataAsMap().get(0)) @@ -127,7 +127,7 @@ public void testSubstrAtom() throws ScriptException { assertThat(context.getAttribute("s5")).isEqualTo(""); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := substr(ds[keep id, string1, string2], 2, 4);"); + Object res = engine.eval("res := substr(ds[keep string1, string2], 2, 4);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "string1", "oto", "string2", ""), @@ -146,7 +146,7 @@ public void testReplaceAtom() throws ScriptException { assertThat(context.getAttribute("r2")).isEqualTo("de"); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := replace(ds[keep id, string1, string2], \"o\", \"O\");"); + Object res = engine.eval("res := replace(ds[keep string1, string2], \"o\", \"O\");"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "string1", "tOtO", "string2", "t"), @@ -182,7 +182,7 @@ public void testInstrAtom() throws ScriptException { assertThat(context.getAttribute("i4")).isEqualTo(0L); context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - Object res = engine.eval("res := instr(ds[keep id, string1, string2], \"o\", 0, 2);"); + Object res = engine.eval("res := instr(ds[keep string1, string2], \"o\", 0, 2);"); assertThat(((Dataset) res).getDataAsMap()) .containsExactlyInAnyOrder( Map.of("id", "Toto", "string1", 4L, "string2", 0L), diff --git a/vtl-prov/src/test/java/fr/insee/vtl/prov/ProvenanceListenerTest.java b/vtl-prov/src/test/java/fr/insee/vtl/prov/ProvenanceListenerTest.java index 3966b7e06..fb2f07326 100644 --- a/vtl-prov/src/test/java/fr/insee/vtl/prov/ProvenanceListenerTest.java +++ b/vtl-prov/src/test/java/fr/insee/vtl/prov/ProvenanceListenerTest.java @@ -95,10 +95,10 @@ public void testWithEmptyLines() throws ScriptException { """ - ds1 := data.ds1[calc identifier id := id, var1 := cast(var1, integer), var2 := cast(var2, integer)]; + ds1 := data.ds1[calc var1 := cast(var1, integer), var2 := cast(var2, integer)]; - ds2_out := other.ds2[calc identifier id := id, var1 := cast(var1, integer), var2 := cast(var2, integer)]; + ds2_out := other.ds2[calc var1 := cast(var1, integer), var2 := cast(var2, integer)]; ds_sum := ds1 + ds2_out; ds_mul <- ds_sum * 3; 'data.ds_res' <- ds_mul[filter mod(var1, 2) = 0][calc var_sum := var1 + var2]; diff --git a/vtl-prov/src/test/java/fr/insee/vtl/prov/RDFTest.java b/vtl-prov/src/test/java/fr/insee/vtl/prov/RDFTest.java index 5560966a6..8d5870e1d 100644 --- a/vtl-prov/src/test/java/fr/insee/vtl/prov/RDFTest.java +++ b/vtl-prov/src/test/java/fr/insee/vtl/prov/RDFTest.java @@ -167,14 +167,13 @@ define datapoint ruleset NUTS3_TYPES (variable facility_type, nb) is CENSUS_NUTS3_2021 := LEGAL_POP [rename REF_AREA to nuts3, TIME_PERIOD to year, POP_TOT to pop] [filter year = "2021"] [calc pop := cast(pop, integer)] - [drop year, NB_COM, POP_MUNI]; + [drop NB_COM, POP_MUNI]; // Extract dataset on general practitioners from BPE by NUTS 3 in 2021 - GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3 [filter facility_type = "D201" and year = "2021"] - [drop facility_type, year]; + GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3 [filter facility_type = "D201" and year = "2021"]; // Merge practitioners and legal population datasets by NUTS 3 in 2021 and compute an indicator - BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021) + BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021 using nuts3) [calc pract_per_10000_inhabitants := nb / pop * 10000] [drop nb, pop];\ """; diff --git a/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java b/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java index 85168f61b..bce7edfa2 100644 --- a/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java +++ b/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java @@ -168,7 +168,7 @@ public void bpeV1() throws ScriptException { "CENSUS_NUTS3_2021 := LEGAL_POP [rename REF_AREA to nuts3, POP_TOT to pop]\n" + "[filter TIME_PERIOD = \"2021\"]\n" + "[calc pop := cast(pop, integer)]\n" - + "[drop TIME_PERIOD, NB_COM, POP_MUNI];"); + + "[drop NB_COM, POP_MUNI];"); Dataset censusNuts2021 = (Dataset) engine.getContext().getAttribute("CENSUS_NUTS3_2021"); Structured.DataStructure censusNuts2021Structure = censusNuts2021.getDataStructure(); @@ -181,8 +181,7 @@ public void bpeV1() throws ScriptException { // Step 7 engine.eval( - "GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3[filter facility_type = \"D201\" and TIME_PERIOD = \"2021\"]\n" - + "[drop facility_type, TIME_PERIOD];"); + "GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3[filter facility_type = \"D201\" and TIME_PERIOD = \"2021\"];"); Dataset generalNuts = (Dataset) engine.getContext().getAttribute("GENERAL_PRACT_NUTS3_2021"); Structured.DataStructure generalNutsStructure = generalNuts.getDataStructure(); @@ -195,7 +194,7 @@ public void bpeV1() throws ScriptException { // Step 8 engine.eval( - "BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021)\n" + "BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021 using nuts3)\n" + "[calc pract_per_10000_inhabitants := nb / pop * 10000]\n" + "[drop nb, pop];"); diff --git a/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java b/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java index c5f96d45f..d716fa4d2 100644 --- a/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java +++ b/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java @@ -16,6 +16,7 @@ import javax.script.ScriptEngineManager; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class SDMXVTLWorkflowTest { @@ -32,6 +33,8 @@ public void setUp() { engine.put(VtlScriptEngine.PROCESSING_ENGINE_NAMES, "spark"); } + // Disabled for now, we have to update the remote resource + @Disabled @Test void testRefFromRepo() { @@ -61,6 +64,9 @@ void testGetEmptyDataset() { .isEqualTo( new Structured.DataStructure( List.of( + new Structured.Component( + "facility_type", String.class, Dataset.Role.IDENTIFIER), + new Structured.Component("TIME_PERIOD", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("nuts3", String.class, Dataset.Role.IDENTIFIER), new Structured.Component( "pract_per_10000_inhabitants", Double.class, Dataset.Role.MEASURE)))); @@ -89,14 +95,16 @@ public void testGetRulesetsVTL() { ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Map.of()); assertThat(sdmxVtlWorkflow.getRulesetsVTL()) - .isEqualTo( + .isEqualToIgnoringWhitespace( "define datapoint ruleset UNIQUE_MUNICIPALITY (variable DEPCOM) is\n" - + " MUNICIPALITY_FORMAT_RULE : match_characters(DEPCOM, \"[0-9]{5}|2[A-B][0-9]{3}\") errorcode \"Municipality code is not in the correct format\"\n" - + " end datapoint ruleset;\n" - + "\n" - + "define datapoint ruleset NUTS3_TYPES (variable facility_type, nb) is\n" - + " BOWLING_ALLEY_RULE : when facility_type = \"F102\" then nb > 10 errorcode \"Not enough bowling alleys\"\n" - + " end datapoint ruleset;"); + + " MUNICIPALITY_FORMAT_RULE : match_characters(DEPCOM, \"[0-9]{5}|2[A-B][0-9]{3}\") errorcode\n" + + " \"Municipality code is not in the correct format\"\n" + + " end datapoint ruleset;\n" + + " \n" + + " define datapoint ruleset NUTS3_TYPES (variable facility_type, nb) is\n" + + " BOWLING_ALLEY_RULE : when facility_type = \"F102\" then nb > 10 errorcode \"Not enough bowling\n" + + " alleys\"\n" + + " end datapoint ruleset;"); } @Test @@ -104,36 +112,38 @@ public void testGetTransformationsVTL() { ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Map.of()); assertThat(sdmxVtlWorkflow.getTransformationsVTL()) - .isEqualTo( + .isEqualToIgnoringWhitespace( "// Validation of municipality code in input file\n" + "CHECK_MUNICIPALITY := check_datapoint(BPE_DETAIL_VTL, UNIQUE_MUNICIPALITY invalid);\n" + "\n" + "// Clean BPE input database\n" - + "BPE_DETAIL_CLEAN := BPE_DETAIL_VTL [drop LAMBERT_X, LAMBERT_Y]\n" - + " [rename ID_EQUIPEMENT to id, TYPEQU to facility_type, DEPCOM to municipality, REF_YEAR to year];\n" + + "BPE_DETAIL_CLEAN := BPE_DETAIL_VTL [drop LAMBERT_X, LAMBERT_Y]\n" + + " [rename ID_EQUIPEMENT to id, TYPEQU to facility_type, DEPCOM to municipality, REF_YEAR to year];\n" + "\n" + "// BPE aggregation by municipality, type and year\n" - + "BPE_MUNICIPALITY <- BPE_DETAIL_CLEAN [aggr nb := count(id) group by municipality, year, facility_type] [rename year to TIME_PERIOD];\n" + + "BPE_MUNICIPALITY <- BPE_DETAIL_CLEAN [aggr nb := count(id) group by municipality, year, facility_type] [rename year\n" + + " to TIME_PERIOD];\n" + "\n" + "// BPE aggregation by NUTS 3, type and year\n" - + "BPE_NUTS3 <- BPE_MUNICIPALITY [calc nuts3 := if substr(municipality,1,2) = \"97\" then substr(municipality,1,3) else substr(municipality,1,2)]\n" - + " [aggr nb := count(nb) group by TIME_PERIOD, nuts3, facility_type];\n" + + "BPE_NUTS3 <- BPE_MUNICIPALITY [calc nuts3 := if substr(municipality,1,2) = \"97\" then substr(municipality,1,3)\n" + + " else substr(municipality,1,2)]\n" + + " [aggr nb := count(nb) group by TIME_PERIOD, nuts3, facility_type];\n" + "\n" + "// BPE validation of facility types by NUTS 3\n" + "CHECK_NUTS3_TYPES := check_datapoint(BPE_NUTS3, NUTS3_TYPES invalid);\n" + "\n" + "// Prepare 2021 census dataset by NUTS 3\n" - + "CENSUS_NUTS3_2021 := LEGAL_POP [rename REF_AREA to nuts3, POP_TOT to pop]\n" - + " [filter TIME_PERIOD = \"2021\"]\n" - + " [calc pop := cast(pop, integer)]\n" - + " [drop TIME_PERIOD, NB_COM, POP_MUNI];\n" + + "CENSUS_NUTS3_2021 := LEGAL_POP [rename REF_AREA to nuts3, POP_TOT to pop]\n" + + " [filter TIME_PERIOD = \"2021\"]\n" + + " [calc pop := cast(pop, integer)]\n" + + " [drop NB_COM, POP_MUNI];\n" + "\n" + "// Extract dataset on general practitioners from BPE by NUTS 3 in 2021\n" - + "GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3 [filter facility_type = \"D201\" and TIME_PERIOD = \"2021\"]\n" - + " [drop facility_type, TIME_PERIOD];\n" + + "GENERAL_PRACT_NUTS3_2021 := BPE_NUTS3 [filter facility_type = \"D201\" and TIME_PERIOD = \"2021\"];\n" + "\n" - + "// Merge practitioners and legal population datasets by NUTS 3 in 2021 and compute an indicator\n" - + "BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021)\n" + + "// Merge practitioners and legal population datasets by NUTS 3 in 2021\n" + + "// and compute an indicator\n" + + "BPE_CENSUS_NUTS3_2021 <- inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021 using nuts3)\n" + " [calc pract_per_10000_inhabitants := nb / pop * 10000]\n" + " [drop nb, pop];"); } diff --git a/vtl-sdmx/src/test/resources/DSD_BPE_CENSUS.xml b/vtl-sdmx/src/test/resources/DSD_BPE_CENSUS.xml index b16ece3e0..1d16e408d 100644 --- a/vtl-sdmx/src/test/resources/DSD_BPE_CENSUS.xml +++ b/vtl-sdmx/src/test/resources/DSD_BPE_CENSUS.xml @@ -1,206 +1,302 @@ - + DSD_BPE_DETAIL_1049 false 2024-03-26T10:49:00Z - - + + + - - - Type d'équipements - La classification des équipements en gammes a pour objectif de réunir des équipements qui présentent des logiques d'implantation voisines, en ce sens qu'ils sont fréquemment présents dans les mêmes communes. Ces regroupements permettent d'élaborer des indicateurs synthétiques reflétant l'organisation hiérarchisée des territoires en termes de services à la population. Les gammes d’équipements sont actualisées chaque année pour une nouvelle version de la Base Permanente des Équipements. En effet, d’une part de nouveaux équipements peuvent être introduits dans la base et, d’autre part, l’implantation des équipements dans les communes peut être modifiée, tout cela pouvant avoir des conséquences sur la composition des gammes. - - POLICE - - - - Code officiel géographique 2021 - Les objets géographiques contenus dans le COG sont les collectivités territoriales (communes, départements, régions, collectivités territoriales à statut particulier), les découpages administratifs (arrondissements, arrondissements municipaux, pseudo-cantons) et certains découpages électoraux (cantons), ainsi que les pays et territoires étrangers. Ils sont répertoriés au 1er janvier de chaque année. Pour la mise à jour du millésime N, tous les événements intervenus entre le 02/01/N-1 et 01/01/N sont pris en compte. - - L'Abergement-Clémenciat - - + + + Type d'équipements + La classification des équipements en gammes a pour objectif de réunir des + équipements qui présentent des logiques d'implantation voisines, en ce sens qu'ils sont fréquemment + présents dans les mêmes communes. Ces regroupements permettent d'élaborer des indicateurs + synthétiques reflétant l'organisation hiérarchisée des territoires en termes de services à la + population. Les gammes d’équipements sont actualisées chaque année pour une nouvelle version de la + Base Permanente des Équipements. En effet, d’une part de nouveaux équipements peuvent être + introduits dans la base et, d’autre part, l’implantation des équipements dans les communes peut être + modifiée, tout cela pouvant avoir des conséquences sur la composition des gammes. + + + POLICE + + + + Code officiel géographique 2021 + Les objets géographiques contenus dans le COG sont les collectivités + territoriales (communes, départements, régions, collectivités territoriales à statut particulier), + les découpages administratifs (arrondissements, arrondissements municipaux, pseudo-cantons) et + certains découpages électoraux (cantons), ainsi que les pays et territoires étrangers. Ils sont + répertoriés au 1er janvier de chaque année. Pour la mise à jour du millésime N, tous les événements + intervenus entre le 02/01/N-1 et 01/01/N sont pris en compte. + + + L'Abergement-Clémenciat + + Code officiel géographique - Les objets géographiques contenus dans le COG sont les collectivités territoriales (communes, départements, régions, collectivités territoriales à statut particulier), les découpages administratifs (arrondissements, arrondissements municipaux, pseudo-cantons) et certains découpages électoraux (cantons), ainsi que les pays et territoires étrangers. Ils sont répertoriés au 1er janvier de chaque année. Pour la mise à jour du millésime N, tous les événements intervenus entre le 02/01/N-1 et 01/01/N sont pris en compte. + Les objets géographiques contenus dans le COG sont les collectivités + territoriales (communes, départements, régions, collectivités territoriales à statut particulier), + les découpages administratifs (arrondissements, arrondissements municipaux, pseudo-cantons) et + certains découpages électoraux (cantons), ainsi que les pays et territoires étrangers. Ils sont + répertoriés au 1er janvier de chaque année. Pour la mise à jour du millésime N, tous les événements + intervenus entre le 02/01/N-1 et 01/01/N sont pris en compte. + Ain - - - - Ensemble des concepts de l'Insee - - Equipement - - - Type de l'équipement - - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) - - - - - Commune - - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) - - - - - Année de référence - - - Abscisse (Système de coordonnées Lambert 93) - - - Ordonnée (Système de coordonnées Lambert 93) - - + + + + Ensemble des concepts de l'Insee + + Equipement + + + Type de l'équipement + + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + + + + + + Commune + + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) + + + + + + Année de référence + + + Abscisse (Système de coordonnées Lambert 93) + + + Ordonnée (Système de coordonnées Lambert 93) + + Code officiel géographique - + Année - + Nombre de communes - + Nombre d'équipement - + Nombre de médecins généralistes pour 10 OOO habitants - + Population municipale - + Population totale - - + + - + Geolocalized Equipments (geography 2021) - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).EQUIPEMENT - - + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).EQUIPEMENT + + + - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) - + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + + - + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).MUNICIPALITY + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).MUNICIPALITY + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) - + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) + + - + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + - + - + - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).LAMBERT_X - - - + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).LAMBERT_X + + + + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).LAMBERT_Y - - - + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).LAMBERT_Y + + + + - Cube populations légales - - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA - - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) - - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TIME_PERIOD - - - - - - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_COM - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).POP_MUNI - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).POP_TOT - - - - + isExternalReference="false" + agencyID="FR1" + id="LEGAL_POP_CUBE" + version="1.0"> + Cube populations légales + + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA + + + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) + + + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TIME_PERIOD + + + + + + + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_COM + + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).POP_MUNI + + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).POP_TOT + + + + + Cube BPE par ville - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).MUNICIPALITY + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).MUNICIPALITY + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_DEPCOM(1.0) + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_EQUIPEMENT + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_EQUIPEMENT + @@ -251,201 +363,246 @@ version="1.0"> Cube BPE par nuts 3 - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).TYPE_EQUIPEMENT + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_TYPEQU(1.0) + - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_YEAR + - + - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_EQUIPEMENT + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_EQUIPEMENT + - + Cube médecins généralistes par habitants ventillé par nuts 3 en 2010 - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).REF_AREA + - urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) + urn:sdmx:org.sdmx.infomodel.codelist.Codelist=FR1:CL_REF_AREA(1.0) + - - - urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_GENERALISTES_PAR_HABITANTS + + + + urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=FR1:INSEE_CONCEPTS(1.0).NB_GENERALISTES_PAR_HABITANTS + - - Dataflow for BPE_DETAIL - urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_DETAIL(1.0) + + Dataflow for BPE_DETAIL + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_DETAIL(1.0) + - - Dataflow for LEGAL_POP_CUBE - urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:LEGAL_POP_CUBE(1.0) + + Dataflow for LEGAL_POP_CUBE + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:LEGAL_POP_CUBE(1.0) + - + Dataflow for BPE_MUNICIPALITY - urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_MUNICIPALITY(1.0) + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_MUNICIPALITY(1.0) + - + Dataflow for BPE_NUTS3 - urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_NUTS3(1.0) + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_NUTS3(1.0) + - + Dataflow for BPE_CENSUS_NUTS3_2021 - urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_CENSUS_NUTS3_2021(1.0) + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=FR1:BPE_CENSUS_NUTS3_2021(1.0) + - - VTL Mapping Scheme #1 - - VTL Mapping #1 - urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=FR1:BPE_DETAIL(1.0) - - - VTL Mapping #2 - urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=FR1:LEGAL_POP_CUBE(1.0) - + + VTL Mapping Scheme #1 + + VTL Mapping #1 + urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=FR1:BPE_DETAIL(1.0) + + + VTL Mapping #2 + urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=FR1:LEGAL_POP_CUBE(1.0) + + - - Ruleset Scheme #1 - - Datapoint Ruleset UNIQUE_MUNICIPALITY - - define datapoint ruleset UNIQUE_MUNICIPALITY (variable DEPCOM) is - MUNICIPALITY_FORMAT_RULE : match_characters(DEPCOM, "[0-9]{5}|2[A-B][0-9]{3}") errorcode "Municipality code is not in the correct format" - end datapoint ruleset; - - - - Datapoint Ruleset NUTS3_TYPES - - define datapoint ruleset NUTS3_TYPES (variable facility_type, nb) is - BOWLING_ALLEY_RULE : when facility_type = "F102" then nb > 10 errorcode "Not enough bowling alleys" - end datapoint ruleset; - - - + + Ruleset Scheme #1 + + Datapoint Ruleset UNIQUE_MUNICIPALITY + + define datapoint ruleset UNIQUE_MUNICIPALITY (variable DEPCOM) is + MUNICIPALITY_FORMAT_RULE : match_characters(DEPCOM, "[0-9]{5}|2[A-B][0-9]{3}") errorcode + "Municipality code is not in the correct format" + end datapoint ruleset; + + + + Datapoint Ruleset NUTS3_TYPES + + define datapoint ruleset NUTS3_TYPES (variable facility_type, nb) is + BOWLING_ALLEY_RULE : when facility_type = "F102" then nb > 10 errorcode "Not enough bowling + alleys" + end datapoint ruleset; + + + - - Transformation Scheme for BPE - CENSUS - - Step 1 - Validation of municipality code in input file - check_datapoint(BPE_DETAIL_VTL, UNIQUE_MUNICIPALITY invalid) - CHECK_MUNICIPALITY - - - Step 2 - Clean BPE input database - - BPE_DETAIL_VTL [drop LAMBERT_X, LAMBERT_Y] - [rename ID_EQUIPEMENT to id, TYPEQU to facility_type, DEPCOM to municipality, REF_YEAR to year] - - BPE_DETAIL_CLEAN - - - Step 3 - BPE aggregation by municipality, type and year - - BPE_DETAIL_CLEAN [aggr nb := count(id) group by municipality, year, facility_type] [rename year to TIME_PERIOD] - - BPE_MUNICIPALITY - - - Step 4 - BPE aggregation by NUTS 3, type and year - - BPE_MUNICIPALITY [calc nuts3 := if substr(municipality,1,2) = "97" then substr(municipality,1,3) else substr(municipality,1,2)] - [aggr nb := count(nb) group by TIME_PERIOD, nuts3, facility_type] - - BPE_NUTS3 - - - Step 5 - BPE validation of facility types by NUTS 3 - check_datapoint(BPE_NUTS3, NUTS3_TYPES invalid) - CHECK_NUTS3_TYPES - - - Step 6 - Prepare 2021 census dataset by NUTS 3 - - LEGAL_POP [rename REF_AREA to nuts3, POP_TOT to pop] - [filter TIME_PERIOD = "2021"] - [calc pop := cast(pop, integer)] - [drop TIME_PERIOD, NB_COM, POP_MUNI] - - CENSUS_NUTS3_2021 - - - Step 7 - Extract dataset on general practitioners from BPE by NUTS 3 in 2021 - - BPE_NUTS3 [filter facility_type = "D201" and TIME_PERIOD = "2021"] - [drop facility_type, TIME_PERIOD] - - GENERAL_PRACT_NUTS3_2021 - - - Step 8 - Merge practitioners and legal population datasets by NUTS 3 in 2021 and compute an indicator - - inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021) + + Transformation Scheme for BPE - CENSUS + + Step 1 + Validation of municipality code in input file + check_datapoint(BPE_DETAIL_VTL, UNIQUE_MUNICIPALITY invalid) + CHECK_MUNICIPALITY + + + Step 2 + Clean BPE input database + + BPE_DETAIL_VTL [drop LAMBERT_X, LAMBERT_Y] + [rename ID_EQUIPEMENT to id, TYPEQU to facility_type, DEPCOM to municipality, REF_YEAR to year] + + BPE_DETAIL_CLEAN + + + Step 3 + BPE aggregation by municipality, type and year + + BPE_DETAIL_CLEAN [aggr nb := count(id) group by municipality, year, facility_type] [rename year + to TIME_PERIOD] + + BPE_MUNICIPALITY + + + Step 4 + BPE aggregation by NUTS 3, type and year + + BPE_MUNICIPALITY [calc nuts3 := if substr(municipality,1,2) = "97" then substr(municipality,1,3) + else substr(municipality,1,2)] + [aggr nb := count(nb) group by TIME_PERIOD, nuts3, facility_type] + + BPE_NUTS3 + + + Step 5 + BPE validation of facility types by NUTS 3 + check_datapoint(BPE_NUTS3, NUTS3_TYPES invalid) + CHECK_NUTS3_TYPES + + + Step 6 + Prepare 2021 census dataset by NUTS 3 + + LEGAL_POP [rename REF_AREA to nuts3, POP_TOT to pop] + [filter TIME_PERIOD = "2021"] + [calc pop := cast(pop, integer)] + [drop NB_COM, POP_MUNI] + + CENSUS_NUTS3_2021 + + + Step 7 + Extract dataset on general practitioners from BPE by NUTS 3 in 2021 + + + BPE_NUTS3 [filter facility_type = "D201" and TIME_PERIOD = "2021"] + + GENERAL_PRACT_NUTS3_2021 + + + Step 8 + Merge practitioners and legal population datasets by NUTS 3 in 2021 + and compute an indicator + + + inner_join(GENERAL_PRACT_NUTS3_2021, CENSUS_NUTS3_2021 using nuts3) [calc pract_per_10000_inhabitants := nb / pop * 10000] [drop nb, pop] - - BPE_CENSUS_NUTS3_2021 - - urn:sdmx:org.sdmx.infomodel.transformation.VtlMappingScheme=FR1:VTLMS1(1.0) - urn:sdmx:org.sdmx.infomodel.transformation.RulesetScheme=FR1:RS1(1.0) - - + + BPE_CENSUS_NUTS3_2021 + + urn:sdmx:org.sdmx.infomodel.transformation.VtlMappingScheme=FR1:VTLMS1(1.0) + + urn:sdmx:org.sdmx.infomodel.transformation.RulesetScheme=FR1:RS1(1.0) + + + \ No newline at end of file diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java index 5cfc73340..d0e97f7dd 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java @@ -130,11 +130,10 @@ public void testParquetMetadataWriting(@TempDir Path tmpDirectory) throws Script context.setAttribute("ds2", dsWithMetadata, ScriptContext.ENGINE_SCOPE); - engine.eval("ds3 := ds2[calc attribute school_id := school_id, identifier year := year];"); + engine.eval("ds3 := ds2[calc identifier year := year];"); SparkDataset dsWithMetadataAndRoles = (SparkDataset) engine.getContext().getAttribute("ds3"); - assertTrue(dsWithMetadataAndRoles.getDataStructure().get("school_id").isAttribute()); assertTrue(dsWithMetadataAndRoles.getDataStructure().get("year").isIdentifier()); } } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java index 3ad9bb9d3..10808118b 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java @@ -47,7 +47,7 @@ public void testOperators() throws ScriptException { engine.eval( "res := ds1#long1; " + "res1 := isnull(ds1); " - + "ds_1 := ds1[keep id, long1, double1]; ds_2 := ds2[keep id, long1, double1]; " + + "ds_1 := ds1[keep long1, double1]; ds_2 := ds2[keep long1, double1]; " + "res2 := ds_1 + ds_2; " + "res3 := ds_1 - ds_2; " + "res4 := ds_1 * ds_2; " @@ -67,7 +67,8 @@ public void testOperators() throws ScriptException { + "res18 := mod(ds_1, 5); " + "res19 := power(ds_1, 5); " + "res20 := log(abs(ds_1), 5); " - + "ds_11 := ds1[keep id, string1, string2]; ds_22 := ds2[keep id, string1][calc string2 := string1]; " + + "ds_11 := ds1[keep string1, string2]; " + + "ds_22 := ds2[keep string1][calc string2 := string1]; " + "res21 := ds_11 || ds_22; "); var res = engine.getContext().getAttribute("res21"); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(String.class); @@ -78,8 +79,8 @@ public void testPlan() throws ScriptException { engine.getContext().setAttribute("ds_1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.getContext().setAttribute("ds_2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval( - "ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " - + "ds2 := ds_2[keep id, long1][rename long1 to bool_var]; " + "ds1 := ds_1[keep long1][rename long1 to bool_var]; " + + "ds2 := ds_2[keep long1][rename long1 to bool_var]; " + "res := if ds1 > ds2 then ds1 else ds2;"); var res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()) diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java index 842758114..5512f86d6 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java @@ -59,7 +59,7 @@ public void testProjection() throws ScriptException { ScriptContext context = engine.getContext(); context.setAttribute("ds", dataset, ScriptContext.ENGINE_SCOPE); - engine.eval("ds1 := ds[keep name, age];"); + engine.eval("ds1 := ds[keep age];"); assertThat(engine.getContext().getAttribute("ds")) .isInstanceOf(fr.insee.vtl.model.Dataset.class); diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java index 19f8eff0d..888bc55b7 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java @@ -713,9 +713,9 @@ public void testValidationSimpleException() throws ScriptException { Dataset DS_r_to_rename = (Dataset) engine.getContext().getAttribute("DS_r_to_rename"); List DS_r_to_renameMeasure = DS_r_to_rename.getDataStructure().values().stream() - .filter(c -> c.isMeasure()) - .map(c -> c.getName()) - .collect(Collectors.toList()); + .filter(Structured.Component::isMeasure) + .map(Structured.Component::getName) + .toList(); assertThat(DS_r_to_renameMeasure.size()).isEqualTo(4); assertThat(DS_r_to_renameMeasure.contains("imbalance")).isTrue(); } @@ -744,8 +744,8 @@ public void serializationCheckDatapointTest() throws ScriptException { "ds1_1 := ds1[calc identifier id := id, long1 := cast(long1, integer), double1 := cast(double1, number), bool1 := cast(bool1, boolean)]; " + "ds2_1 := ds2[calc identifier id := id, long1 := cast(long1, integer), double1 := cast(double1, number), bool1 := cast(bool1, boolean)]; " + "ds_concat := ds1_1#string1 || \" and \" || ds2_1#string1; " - + "ds1_num := ds1_1[keep id, long1, double1]; " - + "ds2_num := ds2_1[keep id, long1, double1]; " + + "ds1_num := ds1_1[keep long1, double1]; " + + "ds2_num := ds2_1[keep long1, double1]; " + "ds_mod := mod(ds1_num, 2); " + "ds_sum := ds1_num + ds2_num; " + "ds_compare := ds1_num = ds2_num; "