Skip to content

Implement remote DFS support for LogicalFileListFiltered #170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 4 commits into
base: HPCC-35827-LogicalFileListFiltered
Choose a base branch
from
+1,910 −61
Draft

Implement remote DFS support for LogicalFileListFiltered #170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
444bc75
HPCC-35827 Add LogicalFileListFiltered filtering and field projection
jakesmith Feb 6, 2026
90b89c5
Review changes.
jakesmith Feb 18, 2026
ff8437d
Initial plan
Copilot Feb 19, 2026
f3567d6
Add remoteDfs support to LogicalFileListFiltered - implementation pha…
Copilot Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 116 additions & 44 deletions dali/base/dadfs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2191,51 +2191,52 @@ struct DFUQFieldInfo
DFUQResultField field;
std::string name;
DFUQResultFieldType type;
std::string attrPath; // Full attribute path (e.g., "Attr/@name" or "@directory")
};

// enum field, name, type
// enum field, name, type, attrPath
static const DFUQFieldInfo dfuqFieldInfos[] =
{
{DFUQResultField::name, "@name", DFUQResultFieldType::stringType},
{DFUQResultField::description, "@description", DFUQResultFieldType::stringType},
{DFUQResultField::nodegroups, "@group", DFUQResultFieldType::stringType},
{DFUQResultField::kind, "@kind", DFUQResultFieldType::stringType},
{DFUQResultField::timemodified, "@modified", DFUQResultFieldType::stringType},
{DFUQResultField::job, "@job", DFUQResultFieldType::stringType},
{DFUQResultField::owner, "@owner", DFUQResultFieldType::stringType},
{DFUQResultField::recordcount, "@DFUSFrecordCount", DFUQResultFieldType::numericType},
{DFUQResultField::origrecordcount, "@recordCount", DFUQResultFieldType::numericType},
{DFUQResultField::recordsize, "@recordSize", DFUQResultFieldType::numericType},
{DFUQResultField::size, "@DFUSFsize", DFUQResultFieldType::numericType},
{DFUQResultField::origsize, "@size", DFUQResultFieldType::numericType},
{DFUQResultField::workunit, "@workunit", DFUQResultFieldType::stringType},
{DFUQResultField::nodegroup, "@DFUSFcluster", DFUQResultFieldType::stringType},
{DFUQResultField::numsubfiles, "@numsubfiles", DFUQResultFieldType::numericType},
{DFUQResultField::accessed, "@accessed", DFUQResultFieldType::stringType},
{DFUQResultField::numparts, "@numparts", DFUQResultFieldType::numericType},
{DFUQResultField::compressedsize, "@compressedSize", DFUQResultFieldType::numericType},
{DFUQResultField::directory, "@directory", DFUQResultFieldType::stringType},
{DFUQResultField::partmask, "@partmask", DFUQResultFieldType::stringType},
{DFUQResultField::superowners, "@superowners", DFUQResultFieldType::stringType},
{DFUQResultField::persistent, "@persistent", DFUQResultFieldType::boolType},
{DFUQResultField::protect, "@protect", DFUQResultFieldType::stringType},
{DFUQResultField::iscompressed, "@compressed", DFUQResultFieldType::boolType},
{DFUQResultField::cost, "@cost", DFUQResultFieldType::floatType},
{DFUQResultField::numDiskReads, "@numDiskReads", DFUQResultFieldType::numericType},
{DFUQResultField::numDiskWrites, "@numDiskWrites", DFUQResultFieldType::numericType},
{DFUQResultField::atRestCost, "@atRestCost", DFUQResultFieldType::floatType},
{DFUQResultField::accessCost, "@accessCost", DFUQResultFieldType::floatType},
{DFUQResultField::maxSkew, "@maxSkew", DFUQResultFieldType::numericType},
{DFUQResultField::minSkew, "@minSkew", DFUQResultFieldType::numericType},
{DFUQResultField::maxSkewPart, "@maxSkewPart", DFUQResultFieldType::numericType},
{DFUQResultField::minSkewPart, "@minSkewPart", DFUQResultFieldType::numericType},
{DFUQResultField::readCost, "@readCost", DFUQResultFieldType::floatType},
{DFUQResultField::writeCost, "@writeCost", DFUQResultFieldType::floatType},
{DFUQResultField::expireDays, "@expireDays", DFUQResultFieldType::numericType},
{DFUQResultField::subfilenames, "@subfilenames", DFUQResultFieldType::stringType},
{DFUQResultField::blockCompressed, "@blockCompressed", DFUQResultFieldType::boolType},
{DFUQResultField::rowCompressed, "@rowCompressed", DFUQResultFieldType::boolType},
{DFUQResultField::includeAll, "includeAll", DFUQResultFieldType::unknown}
{DFUQResultField::name, "@name", DFUQResultFieldType::stringType, "Attr/@name"},
{DFUQResultField::description, "@description", DFUQResultFieldType::stringType, "Attr/@description"},
{DFUQResultField::nodegroups, "@group", DFUQResultFieldType::stringType, "Attr/@group"},
{DFUQResultField::kind, "@kind", DFUQResultFieldType::stringType, "Attr/@kind"},
{DFUQResultField::timemodified, "@modified", DFUQResultFieldType::stringType, "@modified"},
{DFUQResultField::job, "@job", DFUQResultFieldType::stringType, "Attr/@job"},
{DFUQResultField::owner, "@owner", DFUQResultFieldType::stringType, "Attr/@owner"},
{DFUQResultField::recordcount, "@DFUSFrecordCount", DFUQResultFieldType::numericType, ""},
{DFUQResultField::origrecordcount, "@recordCount", DFUQResultFieldType::numericType, "Attr/@recordCount"},
{DFUQResultField::recordsize, "@recordSize", DFUQResultFieldType::numericType, "Attr/@recordSize"},
{DFUQResultField::size, "@DFUSFsize", DFUQResultFieldType::numericType, ""},
{DFUQResultField::origsize, "@size", DFUQResultFieldType::numericType, "Attr/@size"},
{DFUQResultField::workunit, "@workunit", DFUQResultFieldType::stringType, "Attr/@workunit"},
{DFUQResultField::nodegroup, "@DFUSFcluster", DFUQResultFieldType::stringType, ""},
{DFUQResultField::numsubfiles, "@numsubfiles", DFUQResultFieldType::numericType, "Attr/@numsubfiles"},
{DFUQResultField::accessed, "@accessed", DFUQResultFieldType::stringType, "Attr/@accessed"},
{DFUQResultField::numparts, "@numparts", DFUQResultFieldType::numericType, "@numparts"},
{DFUQResultField::compressedsize, "@compressedSize", DFUQResultFieldType::numericType, "Attr/@compressedSize"},
{DFUQResultField::directory, "@directory", DFUQResultFieldType::stringType, "@directory"},
{DFUQResultField::partmask, "@partmask", DFUQResultFieldType::stringType, "@partmask"},
{DFUQResultField::superowners, "@superowners", DFUQResultFieldType::stringType, "SuperOwner"},
{DFUQResultField::persistent, "@persistent", DFUQResultFieldType::boolType, "Attr/@persistent"},
{DFUQResultField::protect, "@protect", DFUQResultFieldType::stringType, "Attr/@protect"},
{DFUQResultField::iscompressed, "@compressed", DFUQResultFieldType::boolType, "Attr/@compressed"},
{DFUQResultField::cost, "@cost", DFUQResultFieldType::floatType, "Attr/@cost"},
{DFUQResultField::numDiskReads, "@numDiskReads", DFUQResultFieldType::numericType, "Attr/@numDiskReads"},
{DFUQResultField::numDiskWrites, "@numDiskWrites", DFUQResultFieldType::numericType, "Attr/@numDiskWrites"},
{DFUQResultField::atRestCost, "@atRestCost", DFUQResultFieldType::floatType, "Attr/@atRestCost"},
{DFUQResultField::accessCost, "@accessCost", DFUQResultFieldType::floatType, "Attr/@accessCost"},
{DFUQResultField::maxSkew, "@maxSkew", DFUQResultFieldType::numericType, "Attr/@maxSkew"},
{DFUQResultField::minSkew, "@minSkew", DFUQResultFieldType::numericType, "Attr/@minSkew"},
{DFUQResultField::maxSkewPart, "@maxSkewPart", DFUQResultFieldType::numericType, "Attr/@maxSkewPart"},
{DFUQResultField::minSkewPart, "@minSkewPart", DFUQResultFieldType::numericType, "Attr/@minSkewPart"},
{DFUQResultField::readCost, "@readCost", DFUQResultFieldType::floatType, "Attr/@readCost"},
{DFUQResultField::writeCost, "@writeCost", DFUQResultFieldType::floatType, "Attr/@writeCost"},
{DFUQResultField::expireDays, "@expireDays", DFUQResultFieldType::numericType, "Attr/@expireDays"},
{DFUQResultField::subfilenames, "@subfilenames", DFUQResultFieldType::stringType, "Attr/@subfilenames"},
{DFUQResultField::blockCompressed, "@blockCompressed", DFUQResultFieldType::boolType, "Attr/@blockCompressed"},
{DFUQResultField::rowCompressed, "@rowCompressed", DFUQResultFieldType::boolType, "Attr/@rowCompressed"},
{DFUQResultField::includeAll, "includeAll", DFUQResultFieldType::unknown, ""}
};
const size_t dfuqFieldInfosCount = sizeof(dfuqFieldInfos)/sizeof(dfuqFieldInfos[0]);
static_assert(dfuqFieldInfosCount == static_cast<size_t>(DFUQResultField::term),
Expand Down Expand Up @@ -2263,6 +2264,50 @@ static const DFUQResultFieldMap dfuResultFieldStringMap = []
return map;
}();

/* Field name to attribute path map: maps simple field names to their full File attribute paths
key = simple field name (e.g., "name", "size", "recordcount")
value = { full-attribute-path, field-type }

The full attribute path is:
- Direct attribute name for File-level attributes (e.g., "@directory", "@numparts")
- "Attr/" + attribute name for File/Attr properties (e.g., "Attr/@name", "Attr/@owner")

This map converts user-specified field names into the actual paths needed to access
the attributes in the File property tree structure.
*/
typedef std::unordered_map<std::string, std::tuple<std::string, DFUQResultField, DFUQResultFieldType>, CaseInsensitiveHash, CaseInsensitiveEqual> DFUQFieldAttrPathMap;

static const DFUQFieldAttrPathMap dfuFieldAttrPathMap = []
{
DFUQFieldAttrPathMap map;

// Derive field names from dfuqFieldInfos using pre-defined attribute paths
// Skip fields not exposed for filtering
for (size_t i = 0; i < dfuqFieldInfosCount; ++i)
{
DFUQResultField field = dfuqFieldInfos[i].field;
assertex(field == (DFUQResultField)i);

if (dfuqFieldInfos[i].attrPath.size() == 0)
continue;

const char *attrName = dfuqFieldInfos[i].name.c_str();
// Strip '@' prefix to get simple field name
const char* fieldName = (attrName[0] == '@') ? attrName + 1 : attrName;

// Use the pre-defined attribute path from dfuqFieldInfos
map.emplace(fieldName, std::make_tuple(dfuqFieldInfos[i].attrPath, field, dfuqFieldInfos[i].type));
}

// Add field name aliases that override the defaults
// "recordcount" -> origrecordcount (the actual file's record count, not @DFUSFrecordCount)
// "size" -> origsize (the actual file's size, not @DFUSFsize)
map["recordcount"] = std::make_tuple("Attr/@recordCount", DFUQResultField::origrecordcount, DFUQResultFieldType::numericType);
map["size"] = std::make_tuple("Attr/@size", DFUQResultField::origsize, DFUQResultFieldType::numericType);

return map;
}();


const char* getDFUQResultFieldName(DFUQResultField field)
{
Expand Down Expand Up @@ -2305,6 +2350,24 @@ const char* getDFUQResultFieldTypeName(DFUQResultFieldType type)
}
}

bool getFileAttributePath(const char *fieldName, StringBuffer &attributePath, DFUQResultField &field, DFUQResultFieldType &type)
{
if (isEmptyString(fieldName))
return false;

// Look up field name in the attribute path map
auto it = dfuFieldAttrPathMap.find(fieldName);
if (it != dfuFieldAttrPathMap.end())
{
attributePath.set(std::get<0>(it->second).c_str());
field = std::get<1>(it->second);
type = std::get<2>(it->second);
return true;
}

return false;
}

static std::vector<DFUQResultField> dfuQResultFieldsToVector(const DFUQResultField *fields, bool includeTerminator)
{
std::vector<DFUQResultField> result;
Expand Down Expand Up @@ -2396,6 +2459,15 @@ struct SerializeFileAttrOptions
// special handling if DFUSFsize is included (which is calculated client side), to also include dependent origsize
fieldsList[static_cast<unsigned>(DFUQResultField::origsize)] = additive;
break;
case DFUQResultField::iscompressed:
fieldsList[static_cast<unsigned>(field)] = additive;
if (additive)
{
// The client uses these to decide if compressed or not.
fieldsList[static_cast<unsigned>(DFUQResultField::kind)] = true;
fieldsList[static_cast<unsigned>(DFUQResultField::blockCompressed)] = true;
}
break;
default:
{
fieldsList[static_cast<unsigned>(field)] = additive;
Expand Down Expand Up @@ -4354,7 +4426,7 @@ protected: friend class CDistributedFilePart;
{
// NB: this is non-standard, for situations where the cluster name is not known,
// which happens where none has been provided/set to the file descriptor,
// and the file descriptor has been built up of parts with ips.
// and the file descriptor has been built up of parts with ips.
// createClusterInfo will perform a reverse lookup to Dali to try to discover
// a group name.
cluster = createClusterInfo(
Expand Down Expand Up @@ -11959,7 +12031,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I
mb.read(lname);
if (version >= 2)
{
unsigned _opts;
unsigned _opts;
mb.read(_opts);
opts = static_cast<GetFileTreeOpts>(_opts);
bool hasUser;
Expand Down Expand Up @@ -14487,7 +14559,7 @@ IPropertyTreeIterator* CDistributedFileDirectory::getLogicalFiles(
{
fieldsWithSorted = dfuQResultFieldsToVector(fields, false);

// NB: could add sort fields here, that are already in fields, that's okay, they will take precedence by being last
// NB: could add sort fields here, that are already in fields, that's okay, they will take precedence by being last
for (unsigned s=0; sortOrder[s] != DFUQResultField::term; s++)
{
DFUQResultField sortField = sortOrder[s] & DFUQResultField::fieldMask;
Expand Down
1 change: 1 addition & 0 deletions dali/base/dadfs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,7 @@ extern da_decl DFUQResultFieldType getDFUQResultFieldType(DFUQResultField field)
extern da_decl DFUQResultField getDFUQResultField(const char *fieldName);
extern da_decl DFUQResultField getDFUQResultFieldAndType(const char *fieldName);
extern da_decl const char* getDFUQResultFieldTypeName(DFUQResultField field);
extern da_decl bool getFileAttributePath(const char *fieldName, StringBuffer &attributePath, DFUQResultField &field, DFUQResultFieldType &type);


/**
Expand Down
Loading