diff --git a/core/transformations.py b/core/transformations.py index 955d5a4..bda22c1 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -299,11 +299,15 @@ def _build_substring_removal_clauses(client: bigquery.Client, source_table: str, # Identify columns that need substring removal subset_columns = [] - for col in all_columns: + for col in subset_columns: # Skip already processed columns if col.lower() in processed_columns: continue - + + # Skip loop variables so they get processed in the loop variable step + if utils.extract_loop_number(col) is not None: + continue + # Check if any substring from constants.SUBSTRINGS_TO_FIX is in the column name if any(substring in col for substring in constants.SUBSTRINGS_TO_FIX): subset_columns.append(col)