Fix backend filtering compatibility with frontend FilterModal component

[Copilot]

Fixes backend filtering functionality to properly support the frontend FilterModal component's data filter requests.

Changes Made

Bug Fixes

1. Added nodeId field to FileFilters DTO - The frontend sends nodeId with each filter request, but the backend DTO was missing this field
2. Fixed file suffix stripping regex - Improved the regex pattern that strips filename suffixes from feature names (e.g., "age (file.csv)" → "age") to handle:
   • Filenames with spaces (e.g., "My File.csv")
   • Filenames with special characters (e.g., "file@2024.csv")
   • Complex filename patterns

Test Coverage

Added 4 comprehensive tests validating the exact filter structure the frontend sends:

• Feature names with file suffix stripping
• Multiple features with file suffixes and operators
• Categorical filters with file suffixes
• Filenames containing spaces

Backend Filter Structure

The backend correctly processes the frontend's nested filter payload:

{
  "multipleFileFilters": [
    {
      "fileName": "data.csv",
      "nodeId": "node123",
      "filters": {
        "operator": "AND",
        "conditions": {
          "age (data.csv)": {
            "conditions": [
              {"type": "greater", "value": 18, "filterType": "continuous"}
            ],
            "operators": []
          }
        }
      }
    }
  ]
}

Testing

• ✅ All 168 backend tests pass
• ✅ Code review completed - no issues found
• ✅ Security scan completed - no vulnerabilities

All changes are backward compatible and fully tested.

Original prompt

This is my fromtend JavaScript code:

---

💡 You can make Copilot smarter by setting up custom instructions, customizing its development environment and configuring Model Context Protocol (MCP) servers. Learn more Copilot coding agent tips in the docs.

[danibanezRepos]

@copilot import React, { useState, useEffect, useRef } from "react";
import OverlayWrapper from "../../Common/OverlayWrapper/overlayWrapper";
import DatePicker from "react-datepicker";
import "react-datepicker/dist/react-datepicker.css";
import Select from "react-select";
import FilterModalStyles from "./filterModal.module.css";
import TooltipPopup from "../../Common/TooltipPopup/tooltipPopup";
import { IoMdClose } from "react-icons/io";
import { toast } from "react-toastify";
import { CSSTransition, TransitionGroup } from "react-transition-group";
import { filterMultipleFiles } from "../../../util/petitionHandler";
import { updateNodeAxiosBaseURL } from "../../../util/nodeAxiosSetup";
import { useNode } from "../../../context/nodeContext";

const initializeFilters = (dataStatistics) => {
if (!dataStatistics) return {};
const filters = {};
const allFeatures = [
...(dataStatistics.categoricalFeatures || []),
...(dataStatistics.continuousFeatures || []),
...(dataStatistics.dateFeatures || []),
];
allFeatures.forEach((feature) => {
filters[feature.featureName] = null;
});
return filters;
};

// Modal that works with the features in dataStatistics. Allows to build a filter query for the backend to repopulate dataResults
const FilterModal = ({ isOpen, dataStatistics, closeModal, filters, setFilters, setFilteredDataStatistics, dataResults, activeFileIndices, setDataResults, combineSelectedData, setDataStatistics }) => {
const [filterConditions, setFilterConditions] = useState({});
const [logicalOperators, setLogicalOperators] = useState({});
const [globalLogicalOperator, setGlobalLogicalOperator] = useState("AND");

const [selectedFeature, setSelectedFeature] = useState("");
const [dateFilterType, setDateFilterType] = useState("equal");
const [dateValue, setDateValue] = useState(new Date());
const [dateRange, setDateRange] = useState([new Date(), new Date()]);
const [continuousFilterType, setContinuousFilterType] = useState("equal");
const [minValue, setMinValue] = useState("");
const [maxValue, setMaxValue] = useState("");
const [selectedCategories, setSelectedCategories] = useState([]);
const [isLoading, setIsLoading] = useState(false);

const addButtonRef = useRef(null);
const [showTooltip, setShowTooltip] = useState(false);
const { selectedNodes } = useNode();

// Initialize local filter placeholders whenever dataStatistics changes
useEffect(() => {
if (dataStatistics) {
setFilterConditions(initializeFilters(dataStatistics));
}
}, [dataStatistics]);

const handleApplyFilters = async () => {
setIsLoading(true);
try {
const multipleFileFilters = dataResults.map((fileResult, idx) => {
const basePayload = {
fileName: fileResult.fileName,
nodeId: fileResult.nodeId
};
if (!activeFileIndices[idx]) {
return { ...basePayload, filters: null };
}
const fileSpecificConditions = {};
for (let feature in filterConditions) {
if (
feature.endsWith((${fileResult.fileName})) &&
filterConditions[feature]?.length > 0
) {
fileSpecificConditions[feature] = {
conditions: filterConditions[feature],
operators: logicalOperators[feature] || [],
};
}
}
if (Object.keys(fileSpecificConditions).length === 0) {
return { ...basePayload, filters: null };
}
return {
...basePayload,
filters: {
conditions: fileSpecificConditions,
operator: globalLogicalOperator,
},
};
});

  const groupedByNode = multipleFileFilters.reduce((acc, filterObj) => {
    const nodeId = filterObj.nodeId || "unknown";
    if (!acc[nodeId]) acc[nodeId] = [];
    acc[nodeId].push(filterObj);
    return acc;
  }, {});

  // For each node group, update the axios baseURL and call filterMultipleFiles
  const allPromises = Object.entries(groupedByNode).map(
    async ([nodeId, filtersForNode]) => {
      // Lookup the node (using selectedNodes)
      const node = selectedNodes.find((n) => n.nodeId === nodeId);
      if (node && node.serviceUrl) {
        updateNodeAxiosBaseURL(node.serviceUrl);
      }
      // Call filtering petition for this node
      return filterMultipleFiles({ multipleFileFilters: filtersForNode });
    }
  );

  // Wait for all responses and flatten them
  const responses = await Promise.all(allPromises);
  const newFilteredResults = responses.flat();

  // Now update the dataResults with new responses based on fileName.
  const updatedResults = dataResults.map((oldItem) => {
    const newItem = newFilteredResults.find((fr) => fr.fileName === oldItem.fileName);
    return newItem ? { ...oldItem, ...newItem } : oldItem;
  });


  setDataResults(updatedResults);
  const combined = combineSelectedData(updatedResults, activeFileIndices);
  setDataStatistics(combined);
  setFilteredDataStatistics(combined);

  // If at least one file was filtered, update filters state.
  const hasFilters = multipleFileFilters.some(
    (file) => file.filters && Object.keys(file.filters.conditions).length > 0
  );
  setFilters(hasFilters ? filterConditions : []);
  resetForm();
  closeModal();
} catch (error) {
  // console.error("Error applying multi-file filters:", error);
  toast.error("Error applying filters: " + error.message);
} finally {
  setIsLoading(false);
}

};

const handleAddFilter = () => {
if (!selectedFeature) {
setShowTooltip(true);
return;
}

let newCondition = null;
let createdCategorical = false;
if (
  dataStatistics?.categoricalFeatures?.some(
    (f) => f.featureName === selectedFeature
  )
) {
  const newConditions = selectedCategories.map((category) => ({
    type: "equal",
    value: category,
    filterType: "categorical",
  }));
  addConditionsToFeature(selectedFeature, newConditions);
  setSelectedCategories([]);
  createdCategorical = newConditions.length > 0;
} else if (
  dataStatistics?.continuousFeatures?.some(
    (f) => f.featureName === selectedFeature
  )
) {
  newCondition = {
    type: continuousFilterType,
    value:
      continuousFilterType === "between"
        ? [minValue, maxValue]
        : minValue,
    filterType: "continuous",
  };
} else if (
  dataStatistics?.dateFeatures?.some(
    (f) => f.featureName === selectedFeature
  )
) {
  newCondition = {
    type: dateFilterType,
    value:
      dateFilterType === "between"
        ? [dateRange[0].toISOString(), dateRange[1].toISOString()]
        : dateValue.toISOString(),
    filterType: "date",
  };
}

if (newCondition) {
  addConditionsToFeature(selectedFeature, [newCondition]);
  setShowTooltip(false);
} else if (createdCategorical) {
  setShowTooltip(false);
} else {
  setShowTooltip(true);
}

};

const addConditionsToFeature = (featureName, newConds) => {
const newFilterConditions = { ...filterConditions };
const newLogicalOperators = { ...logicalOperators };

if (!newFilterConditions[featureName]) {
  newFilterConditions[featureName] = [];
  newLogicalOperators[featureName] = [];
}
const existingLength = newFilterConditions[featureName].length;
newFilterConditions[featureName] = [
  ...newFilterConditions[featureName],
  ...newConds,
];

if (existingLength > 0) {
  newLogicalOperators[featureName] = [
    ...newLogicalOperators[featureName],
    ...new Array(newConds.length).fill("AND"),
  ];
} else if (newConds.length > 1) {
  newLogicalOperators[featureName] = [
    ...newLogicalOperators[featureName],
    ...new Array(newConds.length - 1).fill("AND"),
  ];
}
setFilterConditions(newFilterConditions);
setLogicalOperators(newLogicalOperators);

};

const getCategoryOptions = (featureName) => {
const feature = dataStatistics?.categoricalFeatures?.find(
(f) => f.featureName === featureName
);
if (!feature) return [];
const usedCategories = filterConditions[featureName]
? filterConditions[featureName].map((c) => c.value)
: [];
return Object.keys(feature.categoryCounts)
.filter((cat) => !usedCategories.includes(cat))
.map((cat) => ({ value: cat, label: cat }));
};
const getGridTemplate = (feature, cType, dType) => {
if (
dataStatistics?.categoricalFeatures?.some(
(f) => f.featureName === feature
)
) {
return "3fr 1fr"; // categories + button
} else if (cType === "between" || dType === "between") {
return "1fr 1fr 1fr 1fr"; // range fields + button
}
return "1fr 2fr 1fr";
};

const formatSingleCriteria = (condition, featureName) => {
const isDateFeature = dataStatistics?.dateFeatures?.some(
(f) => f.featureName === featureName
);
const { type, value } = condition;

if (Array.isArray(value)) {
  if (isDateFeature) {
    return (
      <>
        Between{" "}
        <span className={FilterModalStyles.criteriaBox}>
          {new Date(value[0]).toLocaleDateString()}
        </span>{" "}
        and{" "}
        <span className={FilterModalStyles.criteriaBox}>
          {new Date(value[1]).toLocaleDateString()}
        </span>
      </>
    );
  }
  if (type === "between") {
    return (
      <>
        Between{" "}
        <span className={FilterModalStyles.criteriaBox}>{value[0]}</span>{" "}
        and{" "}
        <span className={FilterModalStyles.criteriaBox}>{value[1]}</span>
      </>
    );
  }
  return (
    <span className={FilterModalStyles.criteriaBox}>
      {value.join(", ")}
    </span>
  );
} else {
  let displayVal = isDateFeature
    ? new Date(value).toLocaleDateString()
    : value;
  switch (type) {
    case "equal":
      return (
        <>
          Equal to{" "}
          <span className={FilterModalStyles.criteriaBox}>
            {displayVal}
          </span>
        </>
      );
    case "greater":
      return (
        <>
          Greater than{" "}
          <span className={FilterModalStyles.criteriaBox}>
            {displayVal}
          </span>
        </>
      );
    case "less":
      return (
        <>
          Less than{" "}
          <span className={FilterModalStyles.criteriaBox}>
            {displayVal}
          </span>
        </>
      );
    default:
      return (
        <>
          Unknown filter {type} on {displayVal}
        </>
      );
  }
}

};

const toggleLogicalOperator = (featureName, index) => {
const newOps = { ...logicalOperators };
newOps[featureName][index] =
newOps[featureName][index] === "AND" ? "OR" : "AND";
setLogicalOperators(newOps);
};

const handleDeleteFilter = (feature, index) => {
const newConds = { ...filterConditions };
const newOps = { ...logicalOperators };
newConds[feature].splice(index, 1);
if (newConds[feature].length === 0) {
delete newConds[feature];
delete newOps[feature];
} else {
newOps[feature].splice(index, 1);
}
setFilterConditions(newConds);
setLogicalOperators(newOps);
};

const formatCriteria = (criteria, featureName, ops) => {
return criteria.map((cond, i) => (

{i > 0 && (
<button
className={FilterModalStyles.logicalOperatorButton}
onClick={() => toggleLogicalOperator(featureName, i - 1)}
disabled={isLoading}
>
{ops[i - 1]}

)}
{formatSingleCriteria(cond, featureName)}
<button
className={FilterModalStyles.deleteButton}
onClick={() => handleDeleteFilter(featureName, i)}
disabled={isLoading}
>

));
};

const resetForm = () => {
setSelectedFeature("");
setSelectedCategories([]);
setDateFilterType("equal");
setDateValue(new Date());
setDateRange([new Date(), new Date()]);
setContinuousFilterType("equal");
setMinValue("");
setMaxValue("");
};

const getOptions = (stats) => {
if (!stats) return [];
const opts = [];
(stats.categoricalFeatures || []).forEach((f) =>
opts.push({ value: f.featureName, label: f.featureName })
);
(stats.continuousFeatures || []).forEach((f) =>
opts.push({ value: f.featureName, label: f.featureName })
);
(stats.dateFeatures || []).forEach((f) =>
opts.push({ value: f.featureName, label: f.featureName })
);
return opts;
};

const isAddFilterDisabled = () => {
if (!selectedFeature) return true;
if (
dataStatistics?.categoricalFeatures?.some(
(f) => f.featureName === selectedFeature
)
)
return selectedCategories.length === 0;
if (
dataStatistics?.continuousFeatures?.some(
(f) => f.featureName === selectedFeature
)
) {
if (continuousFilterType === "between") return !minValue || !maxValue;
return !minValue;
}
if (
dataStatistics?.dateFeatures?.some(
(f) => f.featureName === selectedFeature
)
) {
if (dateFilterType === "between") return !dateRange[0] || !dateRange[1];
return !dateValue;
}
return false;
};

const hasAtLeastTwoFilters = () => {
return (
Object.values(filterConditions).filter(
(arr) => arr && arr.length > 0
).length > 1
);
};

if (!dataStatistics) return null;

return (

Filter displayed data

<div
className={
selectedFeature
? FilterModalStyles.formRow
: FilterModalStyles.centeredRow
}
>

<Select
classNamePrefix="react-select"
options={getOptions(dataStatistics)}
onChange={(option) => setSelectedFeature(option.value)}
value={getOptions(dataStatistics).find(
(o) => o.value === selectedFeature
)}
isDisabled={isLoading}
aria-label="Select feature to filter"
inputId="feature-select"
/>
{hasAtLeastTwoFilters() && (

Combine all filters with
<button
className={FilterModalStyles.logicalOperatorButton}
onClick={() =>
setGlobalLogicalOperator(
globalLogicalOperator === "AND" ? "OR" : "AND"
)
}
disabled={isLoading}
aria-label={Global logical operator: ${globalLogicalOperator}. Click to toggle.}
>
{globalLogicalOperator}

)}

      {selectedFeature && (
        <div
          className={FilterModalStyles.filterCriteria}
          style={{
            gridTemplateColumns: getGridTemplate(
              selectedFeature,
              continuousFilterType,
              dateFilterType
            ),
          }}
        >
          {dataStatistics?.categoricalFeatures?.some(
            (f) => f.featureName === selectedFeature
          ) && (
              <div>
                <label>Choose categories:</label>
                <Select
                  isMulti
                  classNamePrefix="react-select"
                  options={getCategoryOptions(selectedFeature)}
                  onChange={(vals) =>
                    setSelectedCategories(vals.map((v) => v.value))
                  }
                  value={getCategoryOptions(selectedFeature).filter((opt) =>
                    selectedCategories.includes(opt.value)
                  )}
                  isDisabled={isLoading}
                />
              </div>
            )}

          {dataStatistics?.continuousFeatures?.some(
            (f) => f.featureName === selectedFeature
          ) && (
              <>
                <div>
                  <label htmlFor="continuous-filter-type">Filter Type</label>
                  <select
                    id="continuous-filter-type"
                    data-testid="continuous-filter-type"
                    className={FilterModalStyles.select}
                    onChange={(e) => setContinuousFilterType(e.target.value)}
                    value={continuousFilterType}
                    disabled={isLoading}
                  >
                    <option value="equal">Equal</option>
                    <option value="between">Between</option>
                    <option value="less">Less than</option>
                    <option value="greater">Greater than</option>
                  </select>
                </div>
                {continuousFilterType === "between" ? (
                  <>
                    <div>
                      <label htmlFor="min-value">Min Value</label>
                      <input
                        id="min-value"
                        data-testid="min-value-input"
                        type="number"
                        className={FilterModalStyles.input}
                        onChange={(e) => setMinValue(e.target.value)}
                        value={minValue}
                        disabled={isLoading}
                      />
                    </div>
                    <div>
                      <label htmlFor="max-value">Max Value</label>
                      <input
                        id="max-value"
                        data-testid="max-value-input"
                        type="number"
                        className={FilterModalStyles.input}
                        onChange={(e) => setMaxValue(e.target.value)}
                        value={maxValue}
                        disabled={isLoading}
                      />
                    </div>
                  </>
                ) : (
                  <div>
                    <label htmlFor="continuous-value">Value</label>
                    <input
                      id="continuous-value"
                      type="number"
                      className={FilterModalStyles.input}
                      onChange={(e) => setMinValue(e.target.value)}
                      value={minValue}
                      disabled={isLoading}
                    />
                  </div>
                )}
              </>
            )}
          {dataStatistics?.dateFeatures?.some(
            (f) => f.featureName === selectedFeature
          ) && (
              <>
                <div>
                  <label htmlFor="date-filter-type">Filter Type</label>
                  <select
                    id="date-filter-type"
                    data-testid="date-filter-type"
                    className={FilterModalStyles.select}
                    onChange={(e) => setDateFilterType(e.target.value)}
                    value={dateFilterType}
                    disabled={isLoading}
                  >
                    <option value="equal">Equal</option>
                    <option value="between">Between</option>
                    <option value="less">Sooner than</option>
                    <option value="greater">Later than</option>
                  </select>
                </div>
                {dateFilterType === "between" ? (
                  <>
                    <div>
                      <label>Start Date</label>
                      <DatePicker
                        selected={dateRange[0]}
                        onChange={(date) =>
                          setDateRange([date, dateRange[1]])
                        }
                        className={FilterModalStyles.datePicker}
                        disabled={isLoading}
                      />
                    </div>
                    <div>
                      <label>End Date</label>
                      <DatePicker
                        selected={dateRange[1]}
                        onChange={(date) =>
                          setDateRange([dateRange[0], date])
                        }
                        className={FilterModalStyles.datePicker}
                        disabled={isLoading}
                      />
                    </div>
                  </>
                ) : (
                  <div>
                    <label>Date</label>
                    <DatePicker
                      selected={dateValue}
                      onChange={(date) => setDateValue(date)}
                      className={FilterModalStyles.datePicker}
                      disabled={isLoading}
                    />
                  </div>
                )}
              </>
            )}

          <button
            className={FilterModalStyles.addButton}
            onClick={() => {
              if (isAddFilterDisabled()) {
                setShowTooltip(true);
              } else {
                handleAddFilter();
              }
            }}
            ref={addButtonRef}
            disabled={isAddFilterDisabled() || isLoading}
          >
            {filterConditions[selectedFeature] &&
              filterConditions[selectedFeature].length > 0
              ? "Add More Criteria"
              : "Add Filter"}
          </button>
          {showTooltip && (
            <TooltipPopup
              message={"Please fill in all fields"}
              buttonRef={addButtonRef}
              onClose={() => setShowTooltip(false)}
            />
          )}
        </div>
      )}
    </div>

    {Object.keys(filterConditions).some((k) => filterConditions[k]) && (
      <div className={FilterModalStyles.filterSummary}>
        <TransitionGroup component="ul">
          {Object.entries(filterConditions).map(([feat, conds]) => {
            if (!conds || conds.length === 0) return null;
            return (
              <CSSTransition
                key={feat}
                timeout={500}
                classNames={{
                  enter: FilterModalStyles.itemEnter,
                  enterActive: FilterModalStyles.itemEnterActive,
                  exit: FilterModalStyles.itemExit,
                  exitActive: FilterModalStyles.itemExitActive,
                }}
              >
                <li
                  style={{
                    display: "flex",
                    width: "100%",
                    justifyContent: "space-between",
                    alignItems: "center",
                  }}
                >
                  <div className={FilterModalStyles.filterSummaryTitle}>
                    <strong>{feat}</strong>
                  </div>
                  <div className={FilterModalStyles.criteriaRow}>
                    {formatCriteria(
                      conds,
                      feat,
                      logicalOperators[feat] || []
                    )}
                  </div>
                </li>
              </CSSTransition>
            );
          })}
        </TransitionGroup>
      </div>
    )}

    <div className={FilterModalStyles.buttonRow}>
      <button
        className={FilterModalStyles.cancelButton}
        onClick={closeModal}
        disabled={isLoading}
      >
        Cancel
      </button>
      <button
        className={FilterModalStyles.applyButton}
        onClick={handleApplyFilters}
        disabled={
          isLoading ||
          (Object.keys(filterConditions).every(
            (key) =>
              !filterConditions[key] || filterConditions[key].length === 0
          ) &&
            Object.keys(filters).length === 0)
        }
      >
        {isLoading ? (
          <>
            <span className={FilterModalStyles.spinner}></span>
            Applying Filters...
          </>
        ) : Object.keys(filterConditions).every(
          (key) =>
            !filterConditions[key] || filterConditions[key].length === 0
        )
          ? Object.keys(filters).length > 0
            ? "Reset Previous Filters"
            : "No Filters Set"
          : "Apply Filters"}
      </button>
    </div>
  </div>
</OverlayWrapper>

);
}

export default FilterModal;

import React, { useState, useCallback, useEffect } from "react";
import { useLocation } from "react-router-dom";
import { CSSTransition, TransitionGroup } from "react-transition-group";
import DiscoveryStyles from "./discovery.module.css";
import FileExplorer from "../../components/Common/FileExplorer/fileExplorer";
import StatisticsDisplay from "../../components/Discovery/StatisticsDisplay/statisticsDisplay";
import ToolTray from "../../components/Discovery/ToolTray/toolTray";
import AggregateDisplay from "../../components/Discovery/AggregatesDisplay/aggregateDisplay";
import FilterModal from "../../components/Discovery/FilterModal/filterModal";
import { ToastContainer, toast } from "react-toastify";
import { getNodeDatasets, processSelectedDatasets, getProcessSelectedDatasetsStatus, getProcessSelectedDatasetsResult } from "../../util/petitionHandler";
import { updateNodeAxiosBaseURL } from "../../util/nodeAxiosSetup";

import { useNode } from "../../context/nodeContext";

// Discovery page for data profiling and statistics visualization
function Discovery() {
const [dataResults, setDataResults] = useState([]);
const [activeFileIndices, setActiveFileIndices] = useState([]);
const [dataStatistics, setDataStatistics] = useState(null);
const [filteredDataStatistics, setFilteredDataStatistics] = useState(null);
const [showIndividualView, toggleShownView] = useState(true);
const [showOutliers, setShowOutliers] = useState(false);
const [isToolTrayOpen, setIsToolTrayOpen] = useState(true);
const [selectedEntry, setSelectedEntry] = useState(null);
const [viewportWidth, setViewportWidth] = useState(window.innerWidth);
const [isFiltersOpen, setIsFiltersopen] = useState(false);
const [filters, setFilters] = useState([]);

const [hasProcessedMappingState, setHasProcessedMappingState] = useState(false);
const { selectedNodes } = useNode();
// eslint-disable-next-line no-unused-vars
const [datasets, setDatasets] = useState([]);
const location = useLocation();
const toggleToolTray = () => setIsToolTrayOpen(!isToolTrayOpen);

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

const pollDiscoveryJob = useCallback(async (jobId, onProgress) => {
while (true) {
const status = await getProcessSelectedDatasetsStatus(jobId);
if (typeof status?.percent === "number") onProgress(status.percent);

  const state = String(status?.state || "").toUpperCase();
  if (state === "DONE") {
    const results = await getProcessSelectedDatasetsResult(jobId);
    return results || [];
  }
  if (state === "ERROR") throw new Error(status.message || "Discovery job failed");

  await sleep(500);
}

}, []);

useEffect(() => {
if (location.state?.elementFiles?.length && !hasProcessedMappingState) {
const elementFiles = location.state.elementFiles;
const nodeMapping = {};
elementFiles.forEach(({ nodeId, fileName }) => {
if (!nodeMapping[nodeId]) nodeMapping[nodeId] = [];
nodeMapping[nodeId].push(fileName);
});
(async () => {
try {
let allResults = [];
await Promise.all(
selectedNodes.map(async (node) => {
const filesForNode = nodeMapping[node.nodeId] || [];
if (filesForNode.length === 0) return;
updateNodeAxiosBaseURL(node.serviceUrl);
const result = await processSelectedDatasets(filesForNode);

          if (result.mode === "sync") {
            const arr = result.results || [];
            allResults = allResults.concat(
              arr.map((r) => ({
                ...r,
                fileName: r.fileName || "Unknown File",
                nodeId: node.nodeId,
                nodeName: node.name,
              }))
            );
            return;
          }

          if (result.mode === "async") {
            const arr = await pollDiscoveryJob(result.jobId, () => { });
            allResults = allResults.concat(
              arr.map((r) => ({
                ...r,
                fileName: r.fileName || "Unknown File",
                nodeId: node.nodeId,
                nodeName: node.name,
              }))
            );
            return;
          }

          throw new Error("Unexpected response from processList");

        })
      );
      if (allResults.length === 0) {
        toast.error("No parsed files returned from backend.");
      }
      setDataResults(allResults);
      setActiveFileIndices(allResults.map(() => true));
    } catch (error) {
      console.error("Error processing parsed element files:", error);
      toast.error("Error processing parsed element files: " + error.message);
    }
  })();
  setHasProcessedMappingState(true);
}

}, [location.state, selectedNodes, dataResults.length, hasProcessedMappingState, pollDiscoveryJob]);

const combineSelectedData = (dataResultsArray, activeIndices) => {
const combined = {
continuousFeatures: [],
categoricalFeatures: [],
dateFeatures: [],
chiSquareTest: [],
covariances: {},
pearsonCorrelations: {},
spearmanCorrelations: {},
omittedFeatures: [],
};

dataResultsArray.forEach((res, idx) => {
  if (!activeIndices[idx]) return;
  const fileLabel = res.fileName || `File #${idx + 1}`;

  res.continuousFeatures?.forEach((item) => {
    const newItem = { ...item };
    newItem.originalName = item.featureName;
    newItem.featureName = `${item.featureName} (${fileLabel})`;
    newItem.fileName = res.fileName || fileLabel;
    combined.continuousFeatures.push(newItem);
  });

  res.categoricalFeatures?.forEach((item) => {
    const newItem = { ...item };
    newItem.originalName = item.featureName;
    newItem.featureName = `${item.featureName} (${fileLabel})`;
    newItem.fileName = res.fileName || fileLabel;
    combined.categoricalFeatures.push(newItem);
  });

  res.dateFeatures?.forEach((item) => {
    const newItem = { ...item };
    newItem.originalName = item.featureName;
    newItem.featureName = `${item.featureName} (${fileLabel})`;
    newItem.fileName = res.fileName || fileLabel;
    combined.dateFeatures.push(newItem);
  });

  if (res.chiSquareTest)
    combined.chiSquareTest = combined.chiSquareTest.concat(res.chiSquareTest);
  if (res.covariances) {
    Object.keys(res.covariances).forEach((k) => {
      combined.covariances[k] = {
        ...(combined.covariances[k] || {}),
        ...res.covariances[k],
      };
    });
  }
  if (res.pearsonCorrelations) {
    Object.keys(res.pearsonCorrelations).forEach((k) => {
      combined.pearsonCorrelations[k] = {
        ...(combined.pearsonCorrelations[k] || {}),
        ...res.pearsonCorrelations[k],
      };
    });
  }
  if (res.spearmanCorrelations) {
    Object.keys(res.spearmanCorrelations).forEach((k) => {
      combined.spearmanCorrelations[k] = {
        ...(combined.spearmanCorrelations[k] || {}),
        ...res.spearmanCorrelations[k],
      };
    });
  }
  if (res.omittedFeatures) {
    combined.omittedFeatures = combined.omittedFeatures.concat(res.omittedFeatures);
  }
});

return combined;

};

useEffect(() => {
if (dataResults.length) {
const combined = combineSelectedData(dataResults, activeFileIndices);
setDataStatistics(combined);
setFilteredDataStatistics(combined);
}
}, [dataResults, activeFileIndices]);

const toggleFileActive = (index) => {
setActiveFileIndices((prev) => {
const newState = [...prev];
if (newState[index] === true) {
const howManyOn = newState.filter(Boolean).length;
if (howManyOn === 1) {
return prev;
}
}
newState[index] = !newState[index];
return newState;
});
};

useEffect(() => {
if (selectedNodes && selectedNodes.length > 0) {
const fetchDatasets = async () => {
try {
const results = await Promise.all(
selectedNodes.map(async (node) => {
updateNodeAxiosBaseURL(node.serviceUrl);
const data = await getNodeDatasets();
return { nodeId: node.nodeId, nodeName: node.name, files: data || [] };
})
);
setDatasets(results);
} catch (error) {
console.error("Error fetching datasets:", error);
}
};
fetchDatasets();
}
}, [selectedNodes]);

useEffect(() => {
const handleResize = () => {
setViewportWidth(window.innerWidth);
};
window.addEventListener("resize", handleResize);
return () => {
window.removeEventListener("resize", handleResize);
};
}, []);

useEffect(() => {
if (viewportWidth > 768) setIsToolTrayOpen(true);
else setIsToolTrayOpen(false);
}, [viewportWidth]);

let preSelected = {};
if (location.state?.elementFiles?.length) {
location.state.elementFiles.forEach(({ nodeId, fileName }) => {
if (!preSelected[nodeId]) preSelected[nodeId] = [];
preSelected[nodeId].push(fileName);
});
}

const handleFilesOpened = (allResults) => {
setDataResults(allResults);
setActiveFileIndices(allResults.map((_, idx) => idx === 0));
};

return (

<FilterModal
isOpen={isFiltersOpen}
dataStatistics={dataStatistics}
filters={filters}
setFilters={setFilters}
setFilteredDataStatistics={setFilteredDataStatistics}
closeModal={() => setIsFiltersopen(false)}
dataResults={dataResults}
activeFileIndices={activeFileIndices}
setDataResults={setDataResults}
combineSelectedData={combineSelectedData}
setDataStatistics={setDataStatistics}
/>
{!filteredDataStatistics && (

)}
<CSSTransition
in={!!filteredDataStatistics}
classNames={{
enter: DiscoveryStyles.toolTrayEnter,
enterActive: DiscoveryStyles.toolTrayEnterActive,
exit: DiscoveryStyles.toolTrayExit,
exitActive: DiscoveryStyles.toolTrayExitActive,
}}
timeout={300}
unmountOnExit
>
<ToolTray
data={dataStatistics}
filteredData={filteredDataStatistics}
setFilteredData={setFilteredDataStatistics}
setData={setDataStatistics}
showOutliers={showOutliers}
setShowOutliers={setShowOutliers}
isToolTrayOpen={isToolTrayOpen}
toggleToolTray={toggleToolTray}
selectedEntry={selectedEntry}
setSelectedEntry={setSelectedEntry}
showIndividualView={showIndividualView}
toggleView={() => toggleShownView((v) => !v)}
filters={filters}
toggleFilters={() => setIsFiltersopen((open) => !open)}
dataResults={dataResults}
activeFileIndices={activeFileIndices}
toggleFileActive={toggleFileActive}
/>

<CSSTransition
in={!!filteredDataStatistics}
classNames={{
enter: DiscoveryStyles.statisticsEnter,
enterActive: DiscoveryStyles.statisticsEnterActive,
exit: DiscoveryStyles.statisticsExit,
exitActive: DiscoveryStyles.statisticsExitActive,
}}
timeout={300}
unmountOnExit
>

{showIndividualView ? (
<CSSTransition
key="statistics"
classNames={{
enter: DiscoveryStyles.fadeEnter,
enterActive: DiscoveryStyles.fadeEnterActive,
exit: DiscoveryStyles.fadeExit,
exitActive: DiscoveryStyles.fadeExitActive,
}}
timeout={200}
>

) : (
<CSSTransition
key="aggregate"
classNames={{
enter: DiscoveryStyles.fadeEnter,
enterActive: DiscoveryStyles.fadeEnterActive,
exit: DiscoveryStyles.fadeExit,
exitActive: DiscoveryStyles.fadeExitActive,
}}
timeout={200}
>

)}

);
}

export default Discovery;

export const filterMultipleFiles = async (payload) => {
// payload = { multipleFileFilters: [ {fileName, filters}, ... ] }
try {
// We send JSON directly, not FormData
const response = await nodeAxiosInstance.post(
/taniwha/api/data/filterByNameList,
payload,
{
headers: {
"Content-Type": "application/json",
},
}
);
// Expect an array of AnalyticsResponseDTO
return response.data;
} catch (error) {
throw error;
}
};

I want you to revise the fultering functionallity of files, based on how the frontend attacjed works. Detect bugs and solve it.

[Copilot]

@copilot import React, { useState, useEffect, useRef } from "react";
import OverlayWrapper from "../../Common/OverlayWrapper/overlayWrapper";
import DatePicker from "react-datepicker";...

I see you've shared React code for a FilterModal component, but this repository (MEDIATA_node) is the backend Java service. The frontend code would be in a different repository.

Could you clarify:

1. Should I be working on a different repository for frontend changes?
2. Is there something specific you'd like me to do with this code?
3. The code snippet appears to be incomplete - could you share the full context?

[danibanezRepos]

@copilot Yes. I want you to revise how the backend works in relation with the fromtend, which IS what im attaching. Special characters are cropping the code, try make sense of It.

[Copilot]

@copilot Yes. I want you to revise how the backend works in relation with the fromtend, which IS what im attaching. Special characters are cropping the code, try make sense of It.

I've completed a comprehensive review of the backend filtering functionality based on your frontend code. Here's what I found and fixed:

Bugs Fixed:

1. Missing nodeId field - The FileFilters DTO was missing the nodeId field that your frontend sends (commit 704c031)

2. File suffix stripping bug - The regex that strips filename suffixes from feature names (e.g., "age (file.csv)" → "age") didn't handle:

   • Filenames with spaces
   • Filenames with special characters
   • Updated to use a more robust pattern (commit 75c64b0)

Validation:

• Added 4 comprehensive tests matching your exact frontend structure
• All 168 backend tests pass successfully
• Code review: No issues found
• Security scan: No vulnerabilities

The backend now correctly handles the nested filter structure your frontend sends, including the conditions, operators, and global operator fields.

Warning

Firewall rules blocked me from connecting to one or more addresses (expand for details)

I tried to connect to the following addresses, but was blocked by firewall rules:

• semantics.inf.um.es
  • Triggering command: /usr/lib/jvm/temurin-17-jdk-amd64/bin/java /usr/lib/jvm/temurin-17-jdk-amd64/bin/java -javaagent:/home/REDACTED/.m2/repository/org/jacoco/org.jacoco.agent/0.8.8/org.jacoco.agent-0.8.8-runtime.jar=destfile=/home/REDACTED/work/MEDIATA_node/MEDIATA_node/target/jacoco.exec -jar /home/REDACTED/work/MEDIATA_node/MEDIATA_node/target/surefire/surefirebooter-20260123163719352_3.jar /home/REDACTED/work/MEDIATA_node/MEDIATA_node/target/surefire 2026-01-23T16-37-19_247-jvmRun1 surefire-20260123163719352_1tmp surefire_0-20260123163719352_2tmp (dns block)

If you need me to access, download, or install something from one of these locations, you can either:

• Configure Actions setup steps to set up my environment, which run before the firewall is enabled
• Add the appropriate URLs or hosts to the custom allowlist in this repository's Copilot coding agent settings (admins only)

[danibanezRepos]

@copilot I don't know the nodeId is used at all. These are the axios interceptors (the frontend is supposed to work on multiple frontends at once, so the node id is a server's id for the frontend)

import axios from "axios";
import config from "../config";

// Axios instance configuration for backend API requests
const axiosInstance = axios.create({
baseURL: config.backendUrl,
timeout: 300000,
headers: {
"Content-Type": "application/json",
},
});

axiosInstance.interceptors.request.use(
(config) => {
const token = localStorage.getItem("jwtToken");
if (token) config.headers["Authorization"] = Bearer ${token};

if (
  config.url.includes("/nodes/connect/info") ||
  config.url.includes("/node/validate")
) {
  const tgt = localStorage.getItem("kerberosTGT");
  if (tgt) config.headers["Kerberos-TGT"] = tgt;
}
return config;

},
(error) => Promise.reject(error)
);

// Function to set up response interceptor
export const setupAxiosInterceptors = (logout) => {
axiosInstance.interceptors.response.use(
(response) => response,
(error) => {
if (
error.response &&
(error.response.status === 401 || error.response.status === 403)
)
logout();
return Promise.reject(error);
}
);
};

export default axiosInstance;

import axios from "axios";

// Axios instance configuration for node-specific API requests
const nodeAxiosInstance = axios.create({
baseURL: "",
timeout: 300000,
headers: {
"Content-Type": "application/json",
},
});

nodeAxiosInstance.interceptors.request.use(
(config) => {
const tokensMapping = JSON.parse(localStorage.getItem("jwtNodeTokens") || "{}");
const token = tokensMapping[config.baseURL];
const sessionToken = localStorage.getItem("jwtToken");

if (token)
  config.headers["Authorization"] = `Bearer ${token}`;
else if (sessionToken)
  config.headers["Authorization"] = `Bearer ${sessionToken}`;
return config;

},
(error) => Promise.reject(error)
);

export const setupNodeAxiosInterceptors = (logout) => {
nodeAxiosInstance.interceptors.response.use(
(response) => {
if (response.data && response.data.jwtNodeToken === "Unauthorized") {
localStorage.removeItem("jwtToken");
localStorage.removeItem("kerberosTGT");
localStorage.removeItem("jwtNodeTokens");
logout();
}
return response;
}, (error) => {
if (error.response && (error.response.status === 401 || error.response.status === 403)) {
// Clear all tokens on error status
localStorage.removeItem("jwtToken");
localStorage.removeItem("kerberosTGT");
localStorage.removeItem("jwtNodeTokens");
logout();
}
return Promise.reject(error);
}
);
};

export const updateNodeAxiosBaseURL = (serviceUrl) => {
console.log(Updating Node Axios baseURL to: ${serviceUrl});
nodeAxiosInstance.defaults.baseURL = serviceUrl;
};

export default nodeAxiosInstance;

import React, { createContext, useContext, useState, useEffect, useCallback } from "react";
import { useNavigate } from "react-router-dom";
import { jwtDecode } from "jwt-decode";
import { useNode } from "./nodeContext";
import { setupNodeAxiosInterceptors } from "../util/nodeAxiosSetup";

// Authentication context provider for managing user login state and JWT tokens

const AuthContext = createContext();

export const AuthProvider = ({ children }) => {
const [isAuthenticated, setIsAuthenticated] = useState(false);
const [isLoading, setIsLoading] = useState(true);
const navigate = useNavigate();
const { clearSelectedNodes } = useNode();

const logout = useCallback(() => {
localStorage.removeItem("jwtToken");
localStorage.removeItem("kerberosTGT");
localStorage.removeItem("jwtNodeTokens");
localStorage.removeItem("selectedNodes");
setIsAuthenticated(false);
clearSelectedNodes();
navigate("/login");
}, [navigate, clearSelectedNodes]);

useEffect(() => {
setupNodeAxiosInterceptors(logout);

const token = localStorage.getItem("jwtToken");
const tgt = localStorage.getItem("kerberosTGT");

if (token && tgt) {
  try {
    const decodedToken = jwtDecode(token);
    if (decodedToken.exp * 1000 > Date.now())
      setIsAuthenticated(true);
    else {
      localStorage.removeItem("jwtToken");
      localStorage.removeItem("kerberosTGT");
      localStorage.removeItem("jwtNodeTokens");
    }
  } catch (error) {
    console.error("Failed to decode token:", error);
  }
} setIsLoading(false);

}, [logout]);

const login = (token, tgt) => {
localStorage.removeItem("jwtNodeTokens");
localStorage.setItem("jwtToken", token);
localStorage.setItem("kerberosTGT", tgt);
setIsAuthenticated(true);
};

return (
<AuthContext.Provider value={{ isAuthenticated, login, logout, isLoading }}>
{children}
</AuthContext.Provider>
);
};

export const useAuth = () => useContext(AuthContext);

import React, { createContext, useContext, useState, useEffect } from "react";

// Node context provider for managing selected data nodes

const NodeContext = createContext();

export const NodeProvider = ({ children }) => {
const [selectedNodes, setSelectedNodes] = useState(() => {
try {
const raw = localStorage.getItem("selectedNodes");
return raw ? JSON.parse(raw) : [];
} catch {
return [];
}
});

useEffect(() => {
const onStorage = (e) => {
if (e.key === "selectedNodes") {
try {
setSelectedNodes(e.newValue ? JSON.parse(e.newValue) : []);
} catch {
setSelectedNodes([]);
}
}
};
window.addEventListener("storage", onStorage);
return () => window.removeEventListener("storage", onStorage);
}, []);

const persist = (nodes) => {
setSelectedNodes(nodes);
try {
if (nodes?.length) {
localStorage.setItem("selectedNodes", JSON.stringify(nodes));
} else {
localStorage.removeItem("selectedNodes");
}
} catch {
// ignore quota / serialization issues
}
};

const selectNode = (node) => persist([node]); // single-node mode
const selectNodes = (nodes) => persist(nodes); // multi-node mode
const clearSelectedNodes = () => persist([]);

return (
<NodeContext.Provider
value={{ selectedNodes, selectNode, selectNodes, clearSelectedNodes }}
>
{children}
</NodeContext.Provider>
);
};

export const useNode = () => useContext(NodeContext);

so, please, revise once again how the front end works, and figure put proper inputs for the backend side. This, for instance, is a frontend petition realized, being sent to two different backend nodes at once:

curl ^"https://stratif.guttmann.tech/taniwha/api/data/filterByNameList^" ^
-H ^"sec-ch-ua-platform: ^^"Android^^"^" ^
-H ^"Authorization: Bearer eyJhbGciOiJIUzM4NCJ9.eyJzdWIiOiJkYW5pYmFuZXoiLCJpYXQiOjE3NjkxODkyMTUsImV4cCI6MTc2OTIwMDAxNX0.MXYZ3k7fbab62NL-vKbCPvYV5JYXnE1Vo8RKzdX2LvvGK7agAL-dEmyiAzZ6k5ix^" ^
-H ^"Referer: https://semantics.inf.um.es/^" ^
-H ^"sec-ch-ua: ^^"Chromium^^";v=^^"142^^", ^^"Google Chrome^^";v=^^"142^^", ^^"Not_A Brand^^";v=^^"99^^"^" ^
-H ^"sec-ch-ua-mobile: ?1^" ^
-H ^"User-Agent: Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36^" ^
-H ^"Accept: application/json, text/plain, /^" ^
-H ^"Content-Type: application/json^" ^
--data-raw ^"^{^^"multipleFileFilters^^":^[^{^\^"fileName^\^":^\^"A.csv^\^",^\^"nodeId^\^":^\^"697255add1fcb02f2e1bd143^\^",^\^"filters^\^":null^}^]^}^"

curl ^"https://treelogic.mediata.dev/taniwha/api/data/filterByNameList^" ^
-H ^"sec-ch-ua-platform: ^^"Android^^"^" ^
-H ^"Authorization: Bearer eyJhbGciOiJIUzM4NCJ9.eyJzdWIiOiJkYW5pYmFuZXoiLCJpYXQiOjE3NjkxODkyMTQsImV4cCI6MTc2OTIwMDAxNH0.uCy6cCLxdR4MTlcyZZ9m24KuQoW9QCRsY74DFQ-375Se_r9PsknRexrPK2Lb7OxG^" ^
-H ^"Referer: https://semantics.inf.um.es/^" ^
-H ^"sec-ch-ua: ^^"Chromium^^";v=^^"142^^", ^^"Google Chrome^^";v=^^"142^^", ^^"Not_A Brand^^";v=^^"99^^"^" ^
-H ^"sec-ch-ua-mobile: ?1^" ^
-H ^"User-Agent: Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36^" ^
-H ^"Accept: application/json, text/plain, /^" ^
-H ^"Content-Type: application/json^" ^
--data-raw ^"^{^^"multipleFileFilters^^":^[^{^\^"fileName^\^":^\^"SCUBA_1.csv^\^",^\^"nodeId^\^":^\^"6972555d2be1f747cea61678^\^",^\^"filters^\^":^{^\^"conditions^\^":^{^\^"Sex (SCUBA_1.csv)^\^":^{^\^"conditions^\^":^[^{^\^"type^\^":^\^"equal^\^",^\^"value^\^":^\^"F^\^",^\^"filterType^\^":^\^"categorical^\^"^}^],^\^"operators^\^":^[^]^}^},^\^"operator^\^":^\^"AND^\^"^}^},^{^\^"fileName^\^":^\^"SCUBA_2.csv^\^",^\^"nodeId^\^":^\^"6972555d2be1f747cea61678^\^",^\^"filters^\^":null^}^]^}^"

also, really importantly, verify the correct behavior of this functionallity on what would be a big-sized .excel file, please

[danibanezRepos]

@copilot retry, I still have tokens left?