[DEV-4124] Implement dirNames fetching for manifest generation and Crowdin upload

src/README.md

@@ -21,6 +21,8 @@ The two workflow inputs that control behaviour are:
 
 Both inputs accept a JSON array, a comma-separated list, or one path per line.
 
+When **neither** input is provided, both scripts fall back to the canonical path list published at <https://static-contents.developer.pagopa.it/it/dirNames.json> (the `dirNames` array). That list is treated as the source of truth: `docs-structure.json` is rebuilt from those paths and the Crowdin upload is limited to the `.md` files reachable from them. The `docs/` directory is never scanned wholesale anymore.
+
 ---
 
 ## Scripts
@@ -31,7 +33,7 @@ Entry point: [`generateDocStructure.ts`](generateDocStructure.ts)
 
 Reads `PATHS_TO_UPLOAD` and `PATHS_TO_DELETE` from the environment and writes `docs-structure.json`.
 
-- **Full scan** (no paths provided): walks the entire `docs/` tree and rebuilds the manifest from scratch.
+- **dirNames rebuild** (no inputs provided): fetches `dirNames.json` and rebuilds the manifest from scratch using only those paths. The existing manifest is discarded so anything no longer listed in `dirNames` is dropped.
 - **Incremental update** (`PATHS_TO_UPLOAD` set): loads the existing manifest and merges only the selected nodes into it, creating any missing intermediate directory nodes.
 - **Deletion** (`PATHS_TO_DELETE` set): loads the existing manifest and removes the targeted nodes.
 
@@ -44,7 +46,7 @@ Entry point: [`generateCrowdinConfig.ts`](generateCrowdinConfig.ts)
 Reads `PATHS_TO_UPLOAD` from the environment and writes `crowdin.yml`.
 
 - When `PATHS_TO_UPLOAD` is set, only the `.md` files under the selected paths are included.
-- When it is empty, all `.md` files under `docs/` are included.
+- When it is empty, the script fetches `dirNames.json` and includes the `.md` files reachable from those paths.
 - `docs-structure.json` is always prepended to the files list so translators can translate folder/file labels.
 - When running on GitHub Actions (`GITHUB_OUTPUT` is set), the script also exposes a `found_files` step output containing a JSON array of the collected markdown paths.
 
@@ -65,6 +67,7 @@ All tree-walking, merging, and path-normalisation logic lives in [`docsStructure
 | `deleteSelectedNode` | Removes a node (and its subtree) from the manifest. |
 | `buildCrowdinFileEntries` | Maps source `.md` paths to Crowdin `source`/`translation` pairs, injecting `%locale%` after `docs/`. |
 | `parseRequestedDocsPaths` | Parses a JSON array, CSV, or newline-separated string into a `string[]`. |
+| `fetchDirNamesPaths` | Fetches the canonical `dirNames` array from `DIR_NAMES_URL` and returns it as `string[]`. |
 
 ---
 

src/docsStructure.ts

@@ -5,6 +5,7 @@ export const DOCS_DIR = 'docs';
 export const CONFIG_FILE = 'crowdin.yml';
 export const DOCS_STRUCTURE_FILE = 'docs-structure.json';
 export const DOCS_STRUCTURE_TRANSLATION_PATH = `${DOCS_DIR}/%locale%/_meta/docs-structure.json`;
+export const DIR_NAMES_URL = 'https://static-contents.developer.pagopa.it/it/dirNames.json';
 
 const IGNORED_DIRECTORY_NAMES = new Set(['.gitbook']);
 
@@ -28,6 +29,7 @@ export interface WriteDocsStructureManifestOptions {
   selectedPaths?: string[];
   pathsToDelete?: string[];
   existingManifestPath?: string;
+  rebuildFromSelectedPaths?: boolean;
 }
 
 function toPosixPath(filePath: string): string {
@@ -553,15 +555,54 @@ export function writeDocsStructureManifest(
 ): DocsStructureManifest {
   const selectedPaths = options.selectedPaths ?? [];
   const pathsToDelete = options.pathsToDelete ?? [];
+  const rootName = path.basename(rootDir);
+  const baseManifest: DocsStructureManifest = options.rebuildFromSelectedPaths
+    ? {
+        version: 1,
+        tree: { [rootName]: createEmptyRootNode(rootDir) },
+      }
+    : readDocsStructureManifest(options.existingManifestPath ?? outputPath, rootDir);
+
   const manifest = selectedPaths.length > 0 || pathsToDelete.length > 0
-    ? mergeManifestWithSelectedPaths(
-        readDocsStructureManifest(options.existingManifestPath ?? outputPath, rootDir),
-        selectedPaths,
-        pathsToDelete,
-        rootDir,
-      )
+    ? mergeManifestWithSelectedPaths(baseManifest, selectedPaths, pathsToDelete, rootDir)
     : collectDocsData(rootDir).manifest;
 
   fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
   return manifest;
 }
+
+export async function fetchDirNamesPaths(url: string = DIR_NAMES_URL): Promise<string[]> {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch dirNames from ${url}: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const payload = (await response.json()) as { dirNames?: unknown };
+
+  if (!Array.isArray(payload.dirNames)) {
+    throw new Error(`Expected a "dirNames" array in the payload at ${url}.`);
+  }
+
+  return payload.dirNames.map((entry, index) => {
+    if (typeof entry !== 'string') {
+      throw new Error(
+        `Invalid "dirNames" entry at index ${index} from ${url}: expected a string but received ${
+          entry === null ? 'null' : typeof entry
+        }.`,
+      );
+    }
+
+    const trimmedEntry = entry.trim();
+
+    if (trimmedEntry.length === 0) {
+      throw new Error(
+        `Invalid "dirNames" entry at index ${index} from ${url}: entry is empty.`,
+      );
+    }
+
+    return trimmedEntry;
+  });
+}

src/generateCrowdinConfig.ts

@@ -1,13 +1,13 @@
-
 import * as fs from 'fs';
 import * as yaml from 'js-yaml';
 import * as os from 'os';
 import {
   buildCrowdinFileEntries,
-  collectDocsData,
   collectSelectedMarkdownFiles,
   CONFIG_FILE,
+  DIR_NAMES_URL,
   DOCS_DIR,
+  fetchDirNamesPaths,
   parseRequestedDocsPaths,
   type CrowdinFileEntry,
 } from './docsStructure';
@@ -18,25 +18,46 @@ interface CrowdinConfig {
   files: CrowdinFileEntry[];
 }
 
-function generateCrowdinConfig() {
+async function generateCrowdinConfig() {
   console.log(`🔍 Scanning the "${DOCS_DIR}" directory...`);
 
   if (!fs.existsSync(DOCS_DIR)) {
     console.error(`❌ Error: The directory "${DOCS_DIR}" does not exist.`);
     process.exit(1);
   }
 
-  const pathsToUpload = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD);
-  const mdFiles = pathsToUpload.length > 0
-    ? collectSelectedMarkdownFiles(pathsToUpload)
-    : collectDocsData().mdFiles;
+  const requestedPathsToUpload = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD);
+  let pathsToUpload = requestedPathsToUpload;
+  let usingDirNames = false;
+
+  if (pathsToUpload.length === 0) {
+    console.log(`🌐 Fetching dirNames from ${DIR_NAMES_URL}...`);
+    try {
+      pathsToUpload = await fetchDirNamesPaths();
+    } catch (error) {
+      console.error('❌ Error while fetching dirNames:', error);
+      process.exit(1);
+    }
 
-  if (pathsToUpload.length > 0) {
+    if (pathsToUpload.length === 0) {
+      console.error('❌ The dirNames payload is empty; nothing to upload.');
+      process.exit(1);
+    }
+
+    usingDirNames = true;
+    console.log(`📥 Received ${pathsToUpload.length} path(s) from dirNames.`);
+  } else {
     console.log(`🎯 Limiting the upload to ${pathsToUpload.length} selected path(s).`);
   }
 
+  const mdFiles = collectSelectedMarkdownFiles(pathsToUpload);
+
   if (mdFiles.length === 0) {
-    console.warn(`⚠️ No .md files found in "${DOCS_DIR}".`);
+    console.warn(
+      usingDirNames
+        ? `⚠️ No .md files found under the paths declared in dirNames.`
+        : `⚠️ No .md files found under the selected paths.`,
+    );
   }
 
   const files: CrowdinFileEntry[] = buildCrowdinFileEntries(mdFiles);
@@ -76,4 +97,7 @@ function generateCrowdinConfig() {
   }
 }
 
-generateCrowdinConfig();
+generateCrowdinConfig().catch((error) => {
+  console.error('❌ Unexpected error while generating crowdin.yml:', error);
+  process.exit(1);
+});

src/generateDocStructure.ts

@@ -1,36 +1,65 @@
 import * as fs from 'fs';
 import {
+  DIR_NAMES_URL,
   DOCS_DIR,
   DOCS_STRUCTURE_FILE,
+  fetchDirNamesPaths,
   parseRequestedDocsPaths,
   writeDocsStructureManifest,
 } from './docsStructure';
 
-function generateDocStructure() {
+async function generateDocStructure() {
   console.log(`🔍 Generating a manifest for the "${DOCS_DIR}" directory...`);
 
   if (!fs.existsSync(DOCS_DIR)) {
     console.error(`❌ Error: The directory "${DOCS_DIR}" does not exist.`);
     process.exit(1);
   }
 
-  const selectedPaths = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD);
+  const requestedSelectedPaths = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD);
   const pathsToDelete = parseRequestedDocsPaths(process.env.PATHS_TO_DELETE);
+  const hasExplicitInputs = requestedSelectedPaths.length > 0 || pathsToDelete.length > 0;
+
+  let selectedPaths = requestedSelectedPaths;
+  let rebuildFromSelectedPaths = false;
+
+  if (!hasExplicitInputs) {
+    console.log(`🌐 Fetching dirNames from ${DIR_NAMES_URL}...`);
+    try {
+      selectedPaths = await fetchDirNamesPaths();
+    } catch (error) {
+      console.error('❌ Error while fetching dirNames:', error);
+      process.exit(1);
+    }
+
+    if (selectedPaths.length === 0) {
+      console.error('❌ The dirNames payload is empty; refusing to rebuild an empty manifest.');
+      process.exit(1);
+    }
+
+    rebuildFromSelectedPaths = true;
+    console.log(`📥 Received ${selectedPaths.length} path(s) from dirNames.`);
+  }
 
   let manifest: ReturnType<typeof writeDocsStructureManifest>;
   try {
     manifest = writeDocsStructureManifest(DOCS_STRUCTURE_FILE, DOCS_DIR, {
       selectedPaths,
       pathsToDelete,
       existingManifestPath: DOCS_STRUCTURE_FILE,
+      rebuildFromSelectedPaths,
     });
   } catch (error) {
     console.error('❌ Error while generating the manifest:', error);
     process.exit(1);
   }
 
-  if (selectedPaths.length > 0) {
-    console.log(`🎯 Incrementally updated manifest from ${selectedPaths.length} selected path(s).`);
+  if (rebuildFromSelectedPaths) {
+    console.log(`🧱 Rebuilt manifest from ${selectedPaths.length} dirNames path(s).`);
+  } else if (requestedSelectedPaths.length > 0) {
+    console.log(
+      `🎯 Incrementally updated manifest from ${requestedSelectedPaths.length} selected path(s).`,
+    );
   }
 
   if (pathsToDelete.length > 0) {
@@ -44,4 +73,7 @@ function generateDocStructure() {
   }
 }
 
-generateDocStructure();
+generateDocStructure().catch((error) => {
+  console.error('❌ Unexpected error while generating the manifest:', error);
+  process.exit(1);
+});

tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
-    "target": "ES2020",
-    "module": "Node16",
-    "moduleResolution": "Node16",
+    "target": "es2025",
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
     "esModuleInterop": true,
     "strict": true,
     "skipLibCheck": true,