fix: loop filter scope

eno-core/src/main/java/fr/insee/eno/core/model/calculated/CalculatedExpression.java

@@ -7,6 +7,7 @@
 import fr.insee.eno.core.model.EnoObject;
 import fr.insee.eno.core.parameter.Format;
 import fr.insee.eno.core.reference.VariableIndex;
+import fr.insee.eno.core.utils.vtl.VtlSyntaxUtils;
 import fr.insee.lunatic.model.flat.LabelType;
 import fr.insee.lunatic.model.flat.LabelTypeEnum;
 import fr.insee.pogues.model.ExpressionType;
@@ -38,7 +39,7 @@ public class CalculatedExpression extends EnoObject {
 
     public static CalculatedExpression defaultExpression() {
         CalculatedExpression res = new CalculatedExpression();
-        res.setValue("true");
+        res.setValue(VtlSyntaxUtils.VTL_TRUE);
         res.setType(LabelTypeEnum.VTL.value());
         return res;
     }

eno-core/src/main/java/fr/insee/eno/core/model/navigation/ComponentFilter.java

@@ -22,7 +22,7 @@
 @Context(format = Format.LUNATIC, type = ConditionFilterType.class)
 public class ComponentFilter extends EnoObject {
 
-    public static final String DEFAULT_FILTER_VALUE = "true";
+    public static final String DEFAULT_FILTER_VALUE = VtlSyntaxUtils.VTL_TRUE;
 
     /** Expression initialized with the default value. */
     @Getter @Setter

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticLoopResolution.java

@@ -5,7 +5,6 @@
 import fr.insee.eno.core.model.EnoIdentifiableObject;
 import fr.insee.eno.core.model.EnoObject;
 import fr.insee.eno.core.model.EnoQuestionnaire;
-import fr.insee.eno.core.model.navigation.Filter;
 import fr.insee.eno.core.model.navigation.LinkedLoop;
 import fr.insee.eno.core.model.navigation.StandaloneLoop;
 import fr.insee.eno.core.model.question.DynamicTableQuestion;
@@ -153,22 +152,7 @@ private void setLunaticLoopFilter(Loop lunaticLoop, fr.insee.eno.core.model.navi
                     "Loop '%s' is empty. This means something went wrong during the mapping or loop resolution.",
                     lunaticLoop.getId()));
         }
-        LunaticLoopFilter.computeAndSetConditionFilter(lunaticLoop);
-        Optional<String> occurrenceFilterExpression = occurrenceFilterExpression(enoLoop);
-        occurrenceFilterExpression.ifPresent(value ->
-                LunaticLoopFilter.removeOccurrenceFilterExpression(lunaticLoop, value));
-    }
-
-    private Optional<String> occurrenceFilterExpression(fr.insee.eno.core.model.navigation.Loop enoLoop) {
-        String filterId = enoLoop.getOccurrenceFilterId();
-        if (filterId == null)
-            return Optional.empty();
-        EnoIdentifiableObject occurrenceFilter = enoIndex.get(filterId);
-        if (!(occurrenceFilter instanceof Filter)) {
-            log.warn("Item if id {} should be a Filter object and not a {}", filterId, occurrenceFilter != null ? occurrenceFilter.getClass().getName() : null);
-            return Optional.empty();
-        }
-        return Optional.of(((Filter) occurrenceFilter).getExpression().getValue());
+        lunaticLoop.setConditionFilter(LunaticLoopFilter.computeConditionFilter(enoLoop, enoQuestionnaire));
     }
 
     /** Lunatic linked loops have an "iterations" property.

eno-core/src/main/java/fr/insee/eno/core/processing/out/steps/lunatic/loop/LunaticLoopFilter.java

@@ -1,88 +1,164 @@
 package fr.insee.eno.core.processing.out.steps.lunatic.loop;
 
-import fr.insee.eno.core.exceptions.business.LunaticLoopException;
+import fr.insee.eno.core.model.EnoQuestionnaire;
+import fr.insee.eno.core.model.calculated.BindingReference;
+import fr.insee.eno.core.model.calculated.CalculatedExpression;
+import fr.insee.eno.core.model.navigation.Filter;
+import fr.insee.eno.core.model.navigation.Loop;
+import fr.insee.eno.core.model.sequence.StructureItemReference;
 import fr.insee.eno.core.utils.vtl.VtlSyntaxUtils;
 import fr.insee.lunatic.model.flat.*;
 
 import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Collectors;
 
+
+/**
+ * {@code LunaticLoopFilter} is a utility class that handles the calculation of filters
+ * applicable to loops (or roundabouts) in a questionnaire, considering the specific rules
+ * of the Lunatic framework.
+ *
+ * <p>This class addresses a complex issue related to the combination of filters and loops:
+ * filters must be correctly applied to loops to avoid scope errors, especially when VTL expressions
+ * are only valid at the individual level or within a subset of the loop. The implemented logic ensures
+ * that only filters strictly included within the loop's scope are considered.</p>
+ *
+ * <h2>General Algorithm</h2>
+ * <ol>
+ *   <li>For each filter in the questionnaire:
+ *     <ul>
+ *       <li>Check if the start and end elements of the loop are strictly included within the filter's scope
+ *           (without coinciding with the filter's boundaries).</li>
+ *       <li>If so, add this filter to the list of filters applicable to the loop.</li>
+ *     </ul>
+ *   </li>
+ *   <li>If no filters are applicable, return a default VTL condition ({@code true}).</li>
+ *   <li>Otherwise, combine the expressions of the applicable filters using AND logic.</li>
+ * </ol>
+ *
+ * <h2>Special Cases</h2>
+ * <ul>
+ *   <li>If the start of the loop coincides with the start of the filter, the filter is not applied
+ *       directly to the loop, but only to its child elements.</li>
+ *   <li>Occurrence filters (whose ID matches the loop's filter ID) are excluded.</li>
+ * </ul>
+ *
+ * <h2>Example Usage</h2>
+ * <pre>
+ *   Loop enoLoop = ...; // Loop to process
+ *   EnoQuestionnaire enoQuestionnaire = ...; // Questionnaire containing the filters
+ *   ConditionFilterType condition = LunaticLoopFilter.computeConditionFilter(enoLoop, enoQuestionnaire);
+ * </pre>
+ */
 public class LunaticLoopFilter {
 
     private LunaticLoopFilter(){
         throw new IllegalStateException("Utility class");
     }
 
-    public static void computeAndSetConditionFilter(Loop lunaticLoop) {
-        Class<? extends ComponentType> loopStructureType = determineLoopStructure(lunaticLoop);
-        List<ConditionFilterType> loopStructureFilters = lunaticLoop.getComponents().stream()
-                .filter(loopStructureType::isInstance)
-                .map(loopStructureType::cast)
-                .map(ComponentType::getConditionFilter)
-                .toList();
-        Optional<ConditionFilterType> computedFilter = computeLoopFilterExpression(loopStructureFilters);
-        if (computedFilter.isEmpty())
-            return;
-        lunaticLoop.setConditionFilter(computedFilter.get());
-    }
 
-    public static void removeOccurrenceFilterExpression(Loop lunaticLoop, String occurrenceFilterExpression) {
-        if (occurrenceFilterExpression.isEmpty())
-            return;
-        String expression = lunaticLoop.getConditionFilter().getValue();
-        lunaticLoop.getConditionFilter().setValue(
-                VtlSyntaxUtils.replaceByTrue(expression, occurrenceFilterExpression)
-        );
-    }
+    /**
+     * Computes the filter condition to apply to a given loop, based on the filters
+     * defined in the questionnaire.
+     *
+     * <p>This method:
+     * <ol>
+     *   <li>Filters the list of questionnaire filters to retain only those strictly included
+     *       within the loop's scope.</li>
+     *   <li>If no filters are applicable, returns a default VTL condition ({@code true}).</li>
+     *   <li>Otherwise, combines the expressions of the applicable filters using AND logic.</li>
+     * </ol>
+     *
+     * @param enoLoop The loop for which to compute the filter condition.
+     * @param enoQuestionnaire The questionnaire containing the filters to evaluate.
+     * @return An instance of {@code ConditionFilterType} representing the combined filter condition,
+     *         or a default condition if no filters are applicable.
+     */
+    public static ConditionFilterType computeConditionFilter(Loop enoLoop, EnoQuestionnaire enoQuestionnaire) {
+
+        List<Filter> filtersForLoop = enoQuestionnaire.getFilters().stream()
+                .filter(filter -> isFilterIncludingLoop(filter, enoLoop))
+                .toList();
 
-    private static Class<? extends ComponentType> determineLoopStructure(Loop lunaticLoop) {
-        if (isLoopOfSequence(lunaticLoop))
-            return Sequence.class;
-        if (isLoopOfSubsequence(lunaticLoop)) {
-            safetyCheck(lunaticLoop);
-            return Subsequence.class;
+        if(filtersForLoop.isEmpty()) {
+            ConditionFilterType defaultConditionFilterType = new ConditionFilterType();
+            defaultConditionFilterType.setValue(VtlSyntaxUtils.VTL_TRUE);
+            defaultConditionFilterType.setType(LabelTypeEnum.VTL);
+            return defaultConditionFilterType;
         }
-        throw new LunaticLoopException(
-                "First element of loop " + lunaticLoop + " is neither a sequence or a subsequence.");
+        return computeLoopFilterExpression(filtersForLoop);
     }
 
-    private static boolean isLoopOfSequence(Loop lunaticLoop) {
-        return lunaticLoop.getComponents().getFirst() instanceof Sequence;
-    }
+    /**
+     * Checks if a given filter strictly includes a loop.
+     *
+     * <p>A filter strictly includes a loop if:
+     * <ul>
+     *   <li>The start and end elements of the loop are present within the filter's scope.</li>
+     *   <li>The start and end elements of the loop do not coincide with the filter's boundaries.</li>
+     *   <li>The filter is not an occurrence filter (whose ID matches the loop's filter ID).</li>
+     * </ul>
+     *
+     * @param filter The filter to check.
+     * @param enoLoop The loop to verify.
+     * @return {@code true} if the filter strictly includes the loop, {@code false} otherwise.
+     */
+    private static boolean isFilterIncludingLoop(Filter filter, Loop enoLoop){
+        String occurrenceFilterId = enoLoop.getOccurrenceFilterId();
 
-    private static boolean isLoopOfSubsequence(Loop lunaticLoop) {
-        return lunaticLoop.getComponents().getFirst() instanceof Subsequence;
-    }
 
-    private static void safetyCheck(Loop lunaticLoop) {
-        if (lunaticLoop.getComponents().stream().anyMatch(Sequence.class::isInstance))
-            throw new LunaticLoopException(
-                    "Loop " + lunaticLoop + " starts on a subsequence, shouldn't contain a sequence.");
+        // do not include occurrence filter
+        if(occurrenceFilterId != null && occurrenceFilterId.equals(filter.getId())) return false;
+        String startLoopElementId = enoLoop.getLoopScope().getFirst().getId();
+        String endLoopElementId = enoLoop.getLoopScope().getLast().getId();
+
+        String startFilterElementId = filter.getFilterScope().getFirst().getId();
+        String endFilterElementId = filter.getFilterScope().getLast().getId();
+
+        // prevent scope calculating error
+        if(startLoopElementId.equals(startFilterElementId) || endLoopElementId.equals(endFilterElementId)) return false;
+
+        boolean isStartOfLoopInsideFilter = false;
+        boolean isEndOfLoopInsideFilter = false;
+        for(StructureItemReference structureItemReference : filter.getFilterScope()){
+            String referenceId = structureItemReference.getId();
+            if(startLoopElementId.equals(referenceId)) isStartOfLoopInsideFilter = true;
+            if(endLoopElementId.equals(referenceId)) isEndOfLoopInsideFilter = true;
+            if(isStartOfLoopInsideFilter && isEndOfLoopInsideFilter) return true;
+        }
+        return false;
     }
 
-    private static Optional<ConditionFilterType> computeLoopFilterExpression(List<ConditionFilterType> loopStructureFilters) {
-        // If any structure component is not filtered, the whole loop will never be filtered.
-        if (loopStructureFilters.stream().anyMatch(Objects::isNull))
-            return Optional.empty();
 
+    /**
+     * Combines the VTL expressions of the filters applicable to a loop into a single expression,
+     * using AND logic.
+     *
+     * <p>This method:
+     * <ol>
+     *   <li>Concatenates the VTL expressions of the filters with the {@code AND} operator.</li>
+     *   <li>Extracts the binding dependencies (VTL variables) from the filters and adds them to the resulting condition.</li>
+     * </ol>
+     *
+     * @param loopStructureFilters List of filters applicable to the loop.
+     * @return An instance of {@code ConditionFilterType} containing the combined VTL expression
+     *         and its binding dependencies.
+     */
+    private static ConditionFilterType computeLoopFilterExpression(List<Filter> loopStructureFilters) {
         ConditionFilterType loopFilter = new ConditionFilterType();
-        String expression = loopStructureFilters.stream()// concatenate VTL expressions
-                .map(LabelType::getValue)
-                .distinct() // don't put the same expression twice
-                .map(VtlSyntaxUtils::removeExtraParenthesis) // to not have double parentheses
-                .map(VtlSyntaxUtils::surroundByParenthesis)
-                .collect(Collectors.joining(" " + VtlSyntaxUtils.OR_KEYWORD + " "));
+        String expression = VtlSyntaxUtils.joinByANDLogicExpression(
+                loopStructureFilters.stream()
+                        .map(Filter::getExpression)
+                        .map(CalculatedExpression::getValue)
+                        .toList()
+        );
         loopFilter.setValue(expression);
         loopFilter.setType(LabelTypeEnum.VTL);
-        loopFilter.setShapeFrom(loopStructureFilters.getFirst().getShapeFrom());
-        List<String> bindingDependencies = loopStructureFilters.stream() // concatenate binding dependencies
-                .flatMap(filter -> filter.getBindingDependencies().stream())
+        List<String> bindingDependencies = loopStructureFilters.stream()
+                .flatMap(filter -> filter.getExpression().getBindingReferences().stream().map(BindingReference::getVariableName))
                 .distinct()
                 .toList();
         loopFilter.setBindingDependencies(bindingDependencies);
-        return Optional.of(loopFilter);
+        return loopFilter;
     }
 
 }

eno-core/src/main/java/fr/insee/eno/core/utils/vtl/VtlSyntaxUtils.java

@@ -9,7 +9,10 @@
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
+import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Utility class that provide methods for analyzing/writing VTL expressions.
@@ -22,7 +25,7 @@ private VtlSyntaxUtils() {}
 
     public static final String LEFT_JOIN_OPERATOR = getVTLTokenName(VtlTokens.LEFT_JOIN);
     public static final String USING_KEYWORD = getVTLTokenName(VtlTokens.USING);
-    public static final String OR_KEYWORD = getVTLTokenName(VtlTokens.OR);
+    public static final String VTL_TRUE = "true";
 
     // ----- VTL syntax methods used in Eno
 
@@ -36,16 +39,24 @@ public static String concatenateStrings(String vtlString1, String vtlString2) {
         return vtlString1 + " " + getVTLTokenName(VtlTokens.CONCAT) + " " + vtlString2;
     }
 
+    public static String joinByANDLogicExpression(List<String> vtlStrings) {
+        if (vtlStrings == null || vtlStrings.isEmpty()) return "";
+
+        return vtlStrings.stream()
+                .filter(Objects::nonNull)
+                .map(VtlSyntaxUtils::removeExtraParenthesis)
+                .map(VtlSyntaxUtils::surroundByParenthesis)
+                .collect(Collectors.joining(" " + getVTLTokenName(VtlTokens.AND) + " "));
+    }
+
+
     /**
      *
-     * @param vtlString1
-     * @param vtlString2
+     * @param vtlStrings
      * @return (vtlString1) and (vtlString2)
      */
-    public static String joinByANDLogicExpression(String vtlString1, String vtlString2){
-        return surroundByParenthesis(removeExtraParenthesis(vtlString1))
-                + " "  + getVTLTokenName(VtlTokens.AND) + " "
-                + surroundByParenthesis(removeExtraParenthesis(vtlString2));
+    public static String joinByANDLogicExpression(String... vtlStrings) {
+        return joinByANDLogicExpression(Arrays.asList(vtlStrings));
     }
 
     /**
@@ -128,7 +139,7 @@ public static String invertBooleanExpression(String expression) {
      * @return VTL expression updated
      */
     public static String replaceByTrue(String expression, String expressionToReplace) {
-        return expression.replace(expressionToReplace, "true");
+        return expression.replace(expressionToReplace, VTL_TRUE);
     }
 
     /** List of Trevas token ids for VTL aggregation operators. */

eno-core/src/test/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticLoopFilterIntegrationTest.java

@@ -33,7 +33,7 @@ void testLoopFilterResolution() throws ParsingException {
                 "integration/ddi/ddi-loop-filter.xml");
 
         // Then
-        assertEquals("(not(FILTRE))", findComponentById(lunaticQuestionnaire, "mf5etm57").get().getConditionFilter().getValue());
+        assertEquals("(nvl(COMBIEN, 0) >3)", findComponentById(lunaticQuestionnaire, "mf5etm57").get().getConditionFilter().getValue());
         assertThat(findComponentById(lunaticQuestionnaire, "mf5evs8l").get().getConditionFilter().getValue())
                 .doesNotContain("not(AGES2 <> 10)");
 

eno-core/src/test/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticLoopResolutionTest.java

@@ -165,7 +165,7 @@ void loopDependencies() {
         @Test
         void loopsConditionFilter() {
             lunaticLoops.forEach(loop ->
-                    assertEquals("(true)", loop.getConditionFilter().getValue()));
+                    assertEquals("true", loop.getConditionFilter().getValue()));
         }
 
         @Test

eno-core/src/test/java/fr/insee/eno/core/processing/out/steps/lunatic/LunaticRoundaboutLoopsTest.java

@@ -71,7 +71,7 @@ void roundaboutProperties() {
             assertEquals(LabelTypeEnum.VTL_MD, roundabout.getLabel().getType());
             assertEquals("\"Roundabout declaration\"", roundabout.getDescription().getValue());
             assertEquals(LabelTypeEnum.VTL_MD, roundabout.getDescription().getType());
-            assertEquals("(true)", roundabout.getConditionFilter().getValue());
+            assertEquals("true", roundabout.getConditionFilter().getValue());
             // roundabout specific ones
             assertEquals("count(FIRST_NAME)", roundabout.getIterations().getValue());
             assertEquals(LabelTypeEnum.VTL, roundabout.getIterations().getType());
@@ -166,7 +166,7 @@ void roundaboutProperties() {
             assertEquals("4", roundabout.getPage());
             assertEquals("\"Roundabout on SS2\"", roundabout.getLabel().getValue());
             assertEquals(LabelTypeEnum.VTL_MD, roundabout.getLabel().getType());
-            assertEquals("(true)", roundabout.getConditionFilter().getValue());
+            assertEquals("true", roundabout.getConditionFilter().getValue());
             // roundabout specific ones
             assertEquals("count(Q1)", roundabout.getIterations().getValue());
             assertEquals(LabelTypeEnum.VTL, roundabout.getIterations().getType());