From 15cac01e33f3479273c2874217a3f93932197908 Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin Date: Fri, 29 May 2026 09:52:18 +0200 Subject: [PATCH 1/6] feat: implement fetching of dirNames for manifest generation and Crowdin upload, enhancing path management --- src/README.md | 7 +++-- src/docsStructure.ts | 37 ++++++++++++++++++---- src/generateCrowdinConfig.ts | 44 +++++++++++++++++++++------ src/generateDocStructure.ts | 42 ++++++++++++++++++++++--- upload_sources_to_crowdin_workflow.md | 4 ++- 5 files changed, 110 insertions(+), 24 deletions(-) diff --git a/src/README.md b/src/README.md index 3c2e735c..fea4e913 100644 --- a/src/README.md +++ b/src/README.md @@ -21,6 +21,8 @@ The two workflow inputs that control behaviour are: Both inputs accept a JSON array, a comma-separated list, or one path per line. +When **neither** input is provided, both scripts fall back to the canonical path list published at (the `dirNames` array). That list is treated as the source of truth: `docs-structure.json` is rebuilt from those paths and the Crowdin upload is limited to the `.md` files reachable from them. The `docs/` directory is never scanned wholesale anymore. + --- ## Scripts @@ -31,7 +33,7 @@ Entry point: [`generateDocStructure.ts`](generateDocStructure.ts) Reads `PATHS_TO_UPLOAD` and `PATHS_TO_DELETE` from the environment and writes `docs-structure.json`. -- **Full scan** (no paths provided): walks the entire `docs/` tree and rebuilds the manifest from scratch. +- **dirNames rebuild** (no inputs provided): fetches `dirNames.json` and rebuilds the manifest from scratch using only those paths. The existing manifest is discarded so anything no longer listed in `dirNames` is dropped. - **Incremental update** (`PATHS_TO_UPLOAD` set): loads the existing manifest and merges only the selected nodes into it, creating any missing intermediate directory nodes. - **Deletion** (`PATHS_TO_DELETE` set): loads the existing manifest and removes the targeted nodes. @@ -44,7 +46,7 @@ Entry point: [`generateCrowdinConfig.ts`](generateCrowdinConfig.ts) Reads `PATHS_TO_UPLOAD` from the environment and writes `crowdin.yml`. - When `PATHS_TO_UPLOAD` is set, only the `.md` files under the selected paths are included. -- When it is empty, all `.md` files under `docs/` are included. +- When it is empty, the script fetches `dirNames.json` and includes the `.md` files reachable from those paths. - `docs-structure.json` is always prepended to the files list so translators can translate folder/file labels. - When running on GitHub Actions (`GITHUB_OUTPUT` is set), the script also exposes a `found_files` step output containing a JSON array of the collected markdown paths. @@ -65,6 +67,7 @@ All tree-walking, merging, and path-normalisation logic lives in [`docsStructure | `deleteSelectedNode` | Removes a node (and its subtree) from the manifest. | | `buildCrowdinFileEntries` | Maps source `.md` paths to Crowdin `source`/`translation` pairs, injecting `%locale%` after `docs/`. | | `parseRequestedDocsPaths` | Parses a JSON array, CSV, or newline-separated string into a `string[]`. | +| `fetchDirNamesPaths` | Fetches the canonical `dirNames` array from `DIR_NAMES_URL` and returns it as `string[]`. | --- diff --git a/src/docsStructure.ts b/src/docsStructure.ts index 010c619b..b417b59c 100644 --- a/src/docsStructure.ts +++ b/src/docsStructure.ts @@ -5,6 +5,7 @@ export const DOCS_DIR = 'docs'; export const CONFIG_FILE = 'crowdin.yml'; export const DOCS_STRUCTURE_FILE = 'docs-structure.json'; export const DOCS_STRUCTURE_TRANSLATION_PATH = `${DOCS_DIR}/%locale%/_meta/docs-structure.json`; +export const DIR_NAMES_URL = 'https://static-contents.developer.pagopa.it/it/dirNames.json'; const IGNORED_DIRECTORY_NAMES = new Set(['.gitbook']); @@ -28,6 +29,7 @@ export interface WriteDocsStructureManifestOptions { selectedPaths?: string[]; pathsToDelete?: string[]; existingManifestPath?: string; + rebuildFromSelectedPaths?: boolean; } function toPosixPath(filePath: string): string { @@ -532,15 +534,38 @@ export function writeDocsStructureManifest( ): DocsStructureManifest { const selectedPaths = options.selectedPaths ?? []; const pathsToDelete = options.pathsToDelete ?? []; + const rootName = path.basename(rootDir); + const baseManifest: DocsStructureManifest = options.rebuildFromSelectedPaths + ? { + version: 1, + tree: { [rootName]: createEmptyRootNode(rootDir) }, + } + : readDocsStructureManifest(options.existingManifestPath ?? outputPath, rootDir); + const manifest = selectedPaths.length > 0 || pathsToDelete.length > 0 - ? mergeManifestWithSelectedPaths( - readDocsStructureManifest(options.existingManifestPath ?? outputPath, rootDir), - selectedPaths, - pathsToDelete, - rootDir, - ) + ? mergeManifestWithSelectedPaths(baseManifest, selectedPaths, pathsToDelete, rootDir) : collectDocsData(rootDir).manifest; fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); return manifest; } + +export async function fetchDirNamesPaths(url: string = DIR_NAMES_URL): Promise { + const response = await fetch(url); + + if (!response.ok) { + throw new Error( + `Failed to fetch dirNames from ${url}: ${response.status} ${response.statusText}`, + ); + } + + const payload = (await response.json()) as { dirNames?: unknown }; + + if (!Array.isArray(payload.dirNames)) { + throw new Error(`Expected a "dirNames" array in the payload at ${url}.`); + } + + return payload.dirNames + .map((entry) => `${entry}`.trim()) + .filter((entry) => entry.length > 0); +} diff --git a/src/generateCrowdinConfig.ts b/src/generateCrowdinConfig.ts index b37b2301..5149bdaa 100644 --- a/src/generateCrowdinConfig.ts +++ b/src/generateCrowdinConfig.ts @@ -1,13 +1,13 @@ - import * as fs from 'fs'; import * as yaml from 'js-yaml'; import * as os from 'os'; import { buildCrowdinFileEntries, - collectDocsData, collectSelectedMarkdownFiles, CONFIG_FILE, + DIR_NAMES_URL, DOCS_DIR, + fetchDirNamesPaths, parseRequestedDocsPaths, type CrowdinFileEntry, } from './docsStructure'; @@ -18,7 +18,7 @@ interface CrowdinConfig { files: CrowdinFileEntry[]; } -function generateCrowdinConfig() { +async function generateCrowdinConfig() { console.log(`🔍 Scanning the "${DOCS_DIR}" directory...`); if (!fs.existsSync(DOCS_DIR)) { @@ -26,17 +26,38 @@ function generateCrowdinConfig() { process.exit(1); } - const pathsToUpload = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD); - const mdFiles = pathsToUpload.length > 0 - ? collectSelectedMarkdownFiles(pathsToUpload) - : collectDocsData().mdFiles; + const requestedPathsToUpload = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD); + let pathsToUpload = requestedPathsToUpload; + let usingDirNames = false; + + if (pathsToUpload.length === 0) { + console.log(`🌐 Fetching dirNames from ${DIR_NAMES_URL}...`); + try { + pathsToUpload = await fetchDirNamesPaths(); + } catch (error) { + console.error('❌ Error while fetching dirNames:', error); + process.exit(1); + } - if (pathsToUpload.length > 0) { + if (pathsToUpload.length === 0) { + console.error('❌ The dirNames payload is empty; nothing to upload.'); + process.exit(1); + } + + usingDirNames = true; + console.log(`📥 Received ${pathsToUpload.length} path(s) from dirNames.`); + } else { console.log(`🎯 Limiting the upload to ${pathsToUpload.length} selected path(s).`); } + const mdFiles = collectSelectedMarkdownFiles(pathsToUpload); + if (mdFiles.length === 0) { - console.warn(`⚠️ No .md files found in "${DOCS_DIR}".`); + console.warn( + usingDirNames + ? `⚠️ No .md files found under the paths declared in dirNames.` + : `⚠️ No .md files found under the selected paths.`, + ); } // Always include docs-structure.json as a source @@ -82,4 +103,7 @@ function generateCrowdinConfig() { } } -generateCrowdinConfig(); +generateCrowdinConfig().catch((error) => { + console.error('❌ Unexpected error while generating crowdin.yml:', error); + process.exit(1); +}); diff --git a/src/generateDocStructure.ts b/src/generateDocStructure.ts index 51bf1379..ab90868f 100644 --- a/src/generateDocStructure.ts +++ b/src/generateDocStructure.ts @@ -1,12 +1,14 @@ import * as fs from 'fs'; import { + DIR_NAMES_URL, DOCS_DIR, DOCS_STRUCTURE_FILE, + fetchDirNamesPaths, parseRequestedDocsPaths, writeDocsStructureManifest, } from './docsStructure'; -function generateDocStructure() { +async function generateDocStructure() { console.log(`🔍 Generating a manifest for the "${DOCS_DIR}" directory...`); if (!fs.existsSync(DOCS_DIR)) { @@ -14,8 +16,30 @@ function generateDocStructure() { process.exit(1); } - const selectedPaths = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD); + const requestedSelectedPaths = parseRequestedDocsPaths(process.env.PATHS_TO_UPLOAD); const pathsToDelete = parseRequestedDocsPaths(process.env.PATHS_TO_DELETE); + const hasExplicitInputs = requestedSelectedPaths.length > 0 || pathsToDelete.length > 0; + + let selectedPaths = requestedSelectedPaths; + let rebuildFromSelectedPaths = false; + + if (!hasExplicitInputs) { + console.log(`🌐 Fetching dirNames from ${DIR_NAMES_URL}...`); + try { + selectedPaths = await fetchDirNamesPaths(); + } catch (error) { + console.error('❌ Error while fetching dirNames:', error); + process.exit(1); + } + + if (selectedPaths.length === 0) { + console.error('❌ The dirNames payload is empty; refusing to rebuild an empty manifest.'); + process.exit(1); + } + + rebuildFromSelectedPaths = true; + console.log(`📥 Received ${selectedPaths.length} path(s) from dirNames.`); + } let manifest: ReturnType; try { @@ -23,14 +47,19 @@ function generateDocStructure() { selectedPaths, pathsToDelete, existingManifestPath: DOCS_STRUCTURE_FILE, + rebuildFromSelectedPaths, }); } catch (error) { console.error('❌ Error while generating the manifest:', error); process.exit(1); } - if (selectedPaths.length > 0) { - console.log(`🎯 Incrementally updated manifest from ${selectedPaths.length} selected path(s).`); + if (rebuildFromSelectedPaths) { + console.log(`🧱 Rebuilt manifest from ${selectedPaths.length} dirNames path(s).`); + } else if (requestedSelectedPaths.length > 0) { + console.log( + `🎯 Incrementally updated manifest from ${requestedSelectedPaths.length} selected path(s).`, + ); } if (pathsToDelete.length > 0) { @@ -44,4 +73,7 @@ function generateDocStructure() { } } -generateDocStructure(); +generateDocStructure().catch((error) => { + console.error('❌ Unexpected error while generating the manifest:', error); + process.exit(1); +}); diff --git a/upload_sources_to_crowdin_workflow.md b/upload_sources_to_crowdin_workflow.md index 93934073..62f9cd67 100644 --- a/upload_sources_to_crowdin_workflow.md +++ b/upload_sources_to_crowdin_workflow.md @@ -24,7 +24,9 @@ Two inputs drive the behavior: - `paths_to_upload` — optional list of docs-relative paths to add/refresh. - `paths_to_delete` — optional list of docs-relative paths to remove from the manifest. -Both accept comma-separated, newline-separated, or JSON array values (see [`parseRequestedDocsPaths`](docsStructure.ts#L385-L412)). +Both accept comma-separated, newline-separated, or JSON array values (see [`parseRequestedDocsPaths`](src/docsStructure.ts)). + +When neither input is provided, both scripts fetch the canonical path list from (the `dirNames` array) via `fetchDirNamesPaths`. That list is the source of truth: `docs-structure.json` is rebuilt from scratch with those paths and the Crowdin upload is scoped to the `.md` files reachable from them. The `docs/` directory is no longer scanned wholesale. --- From bcac948bee708b573cd295a988f00660a2510d47 Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin Date: Fri, 29 May 2026 10:12:49 +0200 Subject: [PATCH 2/6] Use setup-node v6 and remove custom cache with action/cache --- .github/workflows/upload_sources_to_crowdin.yml | 16 ++++------------ tsconfig.json | 6 +++--- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/upload_sources_to_crowdin.yml b/.github/workflows/upload_sources_to_crowdin.yml index d0c8b999..e59b276b 100644 --- a/.github/workflows/upload_sources_to_crowdin.yml +++ b/.github/workflows/upload_sources_to_crowdin.yml @@ -39,21 +39,13 @@ jobs: uses: actions/checkout@v6 - name: Setup Node.JS - uses: ./.github/actions/setup-node - - - name: Cache npm dependencies - id: cache-npm - uses: actions/cache@v5 + uses: actions/setup-node@v6 with: - path: ~/.npm - key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-npm- + node-version: 24 + cache: 'npm' - name: Install dependencies - run: | - npm config set cache ~/.npm - npm ci + run: npm ci - name: Generate docs structure manifest env: diff --git a/tsconfig.json b/tsconfig.json index 5cdde441..752a93e9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { - "target": "ES2020", - "module": "Node16", - "moduleResolution": "Node16", + "target": "es2025", + "module": "nodenext", + "moduleResolution": "nodenext", "esModuleInterop": true, "strict": true, "skipLibCheck": true, From efe960595e39cdcd6aa3de6237931ab8198b0482 Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin Date: Fri, 29 May 2026 16:01:24 +0200 Subject: [PATCH 3/6] fix: update package.json for devportal-docs configuration --- package.json | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 1167d299..f2cf2764 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { - "name": "pagopatranslationspoc", + "name": "devportal-docs", "version": "1.0.0", - "description": "Poc to test crowdin", + "description": "Devportal Docs", "directories": { "doc": "docs" }, @@ -12,14 +12,8 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/Uqido/PagopaTranslationsPOC.git" + "url": "git+https://github.com/pagopa/devportal-docs.git" }, - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/Uqido/PagopaTranslationsPOC/issues" - }, - "homepage": "https://github.com/Uqido/PagopaTranslationsPOC#readme", "dependencies": { "js-yaml": "^4.1.1" }, From e249ae521330718b4e736bdeedf300d581e7e5ec Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin <56671015+Sebastiano-Bertolin@users.noreply.github.com> Date: Fri, 29 May 2026 17:19:41 +0200 Subject: [PATCH 4/6] fix: enhance error handling in fetchDirNamesPaths function --- src/docsStructure.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/docsStructure.ts b/src/docsStructure.ts index b417b59c..42cb9153 100644 --- a/src/docsStructure.ts +++ b/src/docsStructure.ts @@ -565,7 +565,23 @@ export async function fetchDirNamesPaths(url: string = DIR_NAMES_URL): Promise `${entry}`.trim()) - .filter((entry) => entry.length > 0); + return payload.dirNames.map((entry, index) => { + if (typeof entry !== 'string') { + throw new Error( + `Invalid "dirNames" entry at index ${index} from ${url}: expected a string but received ${ + entry === null ? 'null' : typeof entry + }.`, + ); + } + + const trimmedEntry = entry.trim(); + + if (trimmedEntry.length === 0) { + throw new Error( + `Invalid "dirNames" entry at index ${index} from ${url}: entry is empty.`, + ); + } + + return trimmedEntry; + }); } From a94021589db3b3b41a180b8bf769653f9614a7f0 Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin <56671015+Sebastiano-Bertolin@users.noreply.github.com> Date: Fri, 29 May 2026 17:23:03 +0200 Subject: [PATCH 5/6] fix: improve path handling and error management in upload_sources_to_crowdin workflow --- upload_sources_to_crowdin_workflow.md | 59 +++++++++++++++++---------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/upload_sources_to_crowdin_workflow.md b/upload_sources_to_crowdin_workflow.md index 62f9cd67..18cf4488 100644 --- a/upload_sources_to_crowdin_workflow.md +++ b/upload_sources_to_crowdin_workflow.md @@ -85,29 +85,33 @@ ts-node-script --project tsconfig.json generateDocStructure.ts It runs [generateDocStructure.ts](generateDocStructure.ts), which: 1. Verifies `docs/` exists. -2. Parses `PATHS_TO_UPLOAD` and `PATHS_TO_DELETE` via [`parseRequestedDocsPaths`](docsStructure.ts#L385-L412) (accepts JSON array, CSV, or newline list). -3. Calls [`writeDocsStructureManifest`](docsStructure.ts#L519-L538) with those lists. -4. Logs a summary and exits non-zero on failure. +2. Parses `PATHS_TO_UPLOAD` and `PATHS_TO_DELETE` via [`parseRequestedDocsPaths`](src/docsStructure.ts) (accepts JSON array, CSV, or newline list). +3. If both inputs are empty, calls [`fetchDirNamesPaths`](src/docsStructure.ts) to download the canonical path list from `dirNames.json` and flags the run as a rebuild-from-scratch (`rebuildFromSelectedPaths: true`). Aborts with a non-zero exit code if the payload is malformed or empty. +4. Calls [`writeDocsStructureManifest`](src/docsStructure.ts) with the resolved `selectedPaths`, `pathsToDelete`, and rebuild flag. +5. Logs a summary and exits non-zero on failure. #### What `writeDocsStructureManifest` does ```mermaid flowchart TD - A[writeDocsStructureManifest] --> B{selected or
delete paths
present?} - B -- No --> F[collectDocsData
full rescan of docs/] - B -- Yes --> C[readDocsStructureManifest
load existing JSON] - C --> D[mergeManifestWithSelectedPaths] + A[writeDocsStructureManifest] --> B{rebuildFromSelectedPaths?} + B -- Yes --> R[Start from empty root node
tree.docs = empty] + B -- No --> C[readDocsStructureManifest
load existing JSON] + R --> D[mergeManifestWithSelectedPaths
using dirNames paths] + C --> D2[mergeManifestWithSelectedPaths
using user-supplied paths] D --> E[insert/update selected nodes
+ delete requested nodes] - F --> G[Build manifest object
version + tree.docs] - E --> G + D2 --> E + E --> G[Build manifest object
version + tree.docs] G --> H[Write docs-structure.json] ``` -Key helpers in [docsStructure.ts](docsStructure.ts): +Note: `collectDocsData` (a full rescan of `docs/`) is still exported but is no longer reached by this workflow — the manifest is always driven by an explicit list of paths (either the user inputs or the `dirNames` fallback). -- [`buildDirectoryNode`](docsStructure.ts#L112-L142) — recursively walks a directory and produces a node tree where each entry has a human-readable `label` (dashes/underscores turned into spaces) and a `directory` flag. Markdown filenames are added to a flat `mdFiles` list. -- [`collectDocsData`](docsStructure.ts#L443-L461) — full scan: returns both the manifest and the list of all `.md` paths. -- [`readDocsStructureManifest`](docsStructure.ts#L411-L441) — loads the existing JSON, falling back to an empty root node when the file is missing or malformed. +Key helpers in [docsStructure.ts](src/docsStructure.ts): + +- [`buildDirectoryNode`](src/docsStructure.ts) — recursively walks a directory and produces a node tree where each entry has a human-readable `label` (dashes/underscores turned into spaces) and a `directory` flag. Markdown filenames are added to a flat `mdFiles` list. +- [`fetchDirNamesPaths`](src/docsStructure.ts) — downloads `dirNames.json` and returns the validated `dirNames` array; throws on non-string or empty entries so a malformed payload fails the workflow fast. +- [`readDocsStructureManifest`](src/docsStructure.ts) — loads the existing JSON, falling back to an empty root node when the file is missing or malformed. - [`mergeManifestWithSelectedPaths`](docsStructure.ts#L319-L382) — for each selected path: - Normalizes the path, strips a leading `docs/`, rejects empty / ignored (`.gitbook`) entries. - Validates the path stays inside `docs/` and points to a `.md` file or a directory. @@ -174,19 +178,21 @@ The workflow only commits and pushes when `docs-structure.json` has actually cha flowchart TD A[Start] --> B{PATHS_TO_UPLOAD
provided?} B -- Yes --> C[collectSelectedMarkdownFiles
expand dirs to .md leaves] - B -- No --> D[collectDocsData
all .md under docs/] + B -- No --> N[fetchDirNamesPaths
download dirNames.json] + N --> C C --> E[Compose files list:
docs-structure.json
+ buildCrowdinFileEntries] - D --> E E --> F[yaml.dump → crowdin.yml] F --> G{GITHUB_OUTPUT set?} G -- Yes --> H[Append found_files=JSON
step output] G -- No --> I[Skip output
local run] ``` +If `dirNames` is unreachable, malformed, or empty the script aborts with a non-zero exit code, so the Crowdin upload never runs against an undefined source set. + Highlights: -- [`collectSelectedMarkdownFiles`](docsStructure.ts#L463-L505) walks each selected path: if it points to a directory it recurses (skipping `.gitbook`), if it points to a `.md` file it just adds it. Paths are normalized so users can pass `docs/foo/bar.md` or `foo/bar.md` interchangeably. -- [`buildCrowdinFileEntries`](docsStructure.ts#L507-L517) maps each source path to a translation path by injecting `%locale%` after `docs/`. The result looks like: +- [`collectSelectedMarkdownFiles`](src/docsStructure.ts) walks each path (user-supplied or coming from `dirNames`): if it points to a directory it recurses (skipping `.gitbook`), if it points to a `.md` file it just adds it. Paths are normalized so callers can pass `docs/foo/bar.md` or `foo/bar.md` interchangeably. +- [`buildCrowdinFileEntries`](src/docsStructure.ts) maps each source path to a translation path by injecting `%locale%` after `docs/`. The result looks like: ```yaml - source: docs/soluzioni/asilo-nido/README.md @@ -238,6 +244,7 @@ sequenceDiagram participant U as User / Caller workflow participant GA as GitHub Actions runner participant DS as generateDocStructure.ts + participant DN as dirNames.json (static bucket) participant FS as Filesystem (docs/) participant G as Git remote participant CC as generateCrowdinConfig.ts @@ -246,12 +253,22 @@ sequenceDiagram U->>GA: Trigger (optionally with paths_to_upload / paths_to_delete) GA->>GA: Checkout + Node + npm ci GA->>DS: npm run generate_doc_structure - DS->>FS: Read docs/ (full or selected paths) - DS->>FS: Read existing docs-structure.json - DS->>FS: Write merged docs-structure.json + alt No explicit inputs + DS->>DN: fetchDirNamesPaths() + DN-->>DS: dirNames array + DS->>FS: Rebuild docs-structure.json from dirNames paths + else paths_to_upload / paths_to_delete provided + DS->>FS: Read existing docs-structure.json + DS->>FS: Merge selected/deleted paths into manifest + end + DS->>FS: Write docs-structure.json GA->>G: Commit + push (only if file changed) GA->>CC: npm run generate_file - CC->>FS: Collect .md files (full or selected) + alt No explicit inputs + CC->>DN: fetchDirNamesPaths() + DN-->>CC: dirNames array + end + CC->>FS: Collect .md files reachable from those paths CC->>FS: Write crowdin.yml CC->>GA: Set found_files step output GA->>CR: crowdin-action uploads sources from crowdin.yml From 968f72ac9668593db6a2b9fd9023ebf8031c52d7 Mon Sep 17 00:00:00 2001 From: Sebastiano Bertolin <56671015+Sebastiano-Bertolin@users.noreply.github.com> Date: Fri, 29 May 2026 17:37:04 +0200 Subject: [PATCH 6/6] fix: enhance Node.js setup and dependency installation in upload_sources_to_crowdin workflow --- upload_sources_to_crowdin_workflow.md | 50 +++++++++++++-------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/upload_sources_to_crowdin_workflow.md b/upload_sources_to_crowdin_workflow.md index 329db96b..f930336b 100644 --- a/upload_sources_to_crowdin_workflow.md +++ b/upload_sources_to_crowdin_workflow.md @@ -11,8 +11,8 @@ The workflow uploads Markdown source files from `docs/` (plus a `docs-structure. ```mermaid flowchart TD A[Trigger: workflow_dispatch or workflow_call] --> B[Checkout repo] - B --> C[Setup Node.js] - C --> D[Cache + install npm deps] + B --> C[Setup Node.js
with built-in npm cache] + C --> D[Install npm deps
npm ci] D --> E[generate_doc_structure
updates docs-structure.json] E --> F[Commit docs-structure.json
if changed] F --> G[generate_file
writes crowdin.yml] @@ -47,24 +47,22 @@ flowchart LR ## 3. Step-by-step breakdown -### 3.1 Checkout + Node setup + dependency cache +### 3.1 Checkout + Node setup + install ```yaml -- uses: actions/checkout@v6 -- uses: ./.github/actions/setup-node -- uses: actions/cache@v5 +- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd +- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e with: - path: ~/.npm - key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} + node-version: 24 + cache: 'npm' - run: npm ci ``` Standard preparation: -1. Check out the branch that triggered the workflow. -2. Run the local composite action `setup-node` (Node version pinned for the project). -3. Restore the npm cache keyed on `package-lock.json`, so unchanged lockfiles skip a full install. -4. `npm ci` installs the exact dependency tree required by the TypeScript scripts (`ts-node`, `typescript`, `js-yaml`, `@types/*`). +1. Check out the branch that triggered the workflow. The action is pinned by commit SHA. +2. Set up Node.js 24 using `actions/setup-node` (also pinned by SHA). The `cache: 'npm'` option enables the action's built-in npm cache keyed on `package-lock.json`, so unchanged lockfiles skip a full install — no separate `actions/cache` step is needed. +3. `npm ci` installs the exact dependency tree required by the TypeScript scripts (`ts-node`, `typescript`, `js-yaml`, `@types/*`). ### 3.2 Generate the docs structure manifest @@ -76,13 +74,13 @@ Standard preparation: run: npm run generate_doc_structure ``` -`npm run generate_doc_structure` is defined in [package.json](package.json#L8-L11) as: +`npm run generate_doc_structure` is defined in [package.json](package.json#L10-L10) as: ```text -ts-node-script --project tsconfig.json generateDocStructure.ts +ts-node-script --project tsconfig.json src/generateDocStructure.ts ``` -It runs [generateDocStructure.ts](generateDocStructure.ts), which: +It runs [src/generateDocStructure.ts](src/generateDocStructure.ts), which: 1. Verifies `docs/` exists. 2. Parses `PATHS_TO_UPLOAD` and `PATHS_TO_DELETE` via [`parseRequestedDocsPaths`](src/docsStructure.ts) (accepts JSON array, CSV, or newline list). @@ -112,11 +110,11 @@ Key helpers in [docsStructure.ts](src/docsStructure.ts): - [`buildDirectoryNode`](src/docsStructure.ts) — recursively walks a directory and produces a node tree where each entry has a human-readable `label` (dashes/underscores turned into spaces) and a `directory` flag. Markdown filenames are added to a flat `mdFiles` list. - [`fetchDirNamesPaths`](src/docsStructure.ts) — downloads `dirNames.json` and returns the validated `dirNames` array; throws on non-string or empty entries so a malformed payload fails the workflow fast. - [`readDocsStructureManifest`](src/docsStructure.ts) — loads the existing JSON, falling back to an empty root node when the file is missing or malformed. -- [`mergeManifestWithSelectedPaths`](docsStructure.ts#L319-L382) — for each selected path: +- [`mergeManifestWithSelectedPaths`](src/docsStructure.ts) — for each selected path: - Normalizes the path, strips a leading `docs/`, rejects empty / ignored (`.gitbook`) entries. - Validates the path stays inside `docs/` and points to a `.md` file or a directory. - - Builds a node from the filesystem ([`createNodeFromSelectedEntry`](docsStructure.ts#L212-L222)) and inserts it into the existing tree via [`insertSelectedNode`](docsStructure.ts#L224-L268), creating missing intermediate directory nodes on the fly and merging children via [`mergeNodes`](docsStructure.ts#L185-L210). - - For deletions, [`deleteSelectedNode`](docsStructure.ts#L270-L296) walks down the tree and removes the targeted child. + - Builds a node from the filesystem ([`createNodeFromSelectedEntry`](src/docsStructure.ts)) and inserts it into the existing tree via [`insertSelectedNode`](src/docsStructure.ts), creating missing intermediate directory nodes on the fly and merging children via [`mergeNodes`](src/docsStructure.ts). + - For deletions, [`deleteSelectedNode`](src/docsStructure.ts) walks down the tree and removes the targeted child. - Finally `JSON.stringify(manifest, null, 2)` is written to `docs-structure.json` with a trailing newline. #### Manifest shape @@ -172,7 +170,7 @@ The workflow only commits and pushes when `docs-structure.json` has actually cha run: npm run generate_file ``` -`npm run generate_file` runs [generateCrowdinConfig.ts](src/generateCrowdinConfig.ts): +`npm run generate_file` runs [src/generateCrowdinConfig.ts](src/generateCrowdinConfig.ts): ```mermaid flowchart TD @@ -214,7 +212,7 @@ Highlights: ### 3.5 Upload to Crowdin ```yaml -- uses: crowdin/github-action@v2 +- uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 with: upload_sources: true upload_translations: false @@ -228,7 +226,7 @@ Highlights: CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} ``` -The official Crowdin action reads the freshly generated `crowdin.yml` and: +The official Crowdin action (pinned by commit SHA) reads the freshly generated `crowdin.yml` and: - Uploads every file listed under `files:` as a source string set. - Auto-approves imported strings so existing translations get reused immediately. @@ -281,9 +279,9 @@ sequenceDiagram | Concern | Location | | --- | --- | | Workflow definition | [.github/workflows/upload_sources_to_crowdin.yml](.github/workflows/upload_sources_to_crowdin.yml) | -| Manifest entry point | [generateDocStructure.ts](src/generateDocStructure.ts) | -| Crowdin config entry point | [generateCrowdinConfig.ts](src/generateCrowdinConfig.ts) | -| Shared helpers (tree walk, merge, parsing) | [docsStructure.ts](src/docsStructure.ts) | +| Manifest entry point | [src/generateDocStructure.ts](src/generateDocStructure.ts) | +| Crowdin config entry point | [src/generateCrowdinConfig.ts](src/generateCrowdinConfig.ts) | +| Shared helpers (tree walk, merge, parsing) | [src/docsStructure.ts](src/docsStructure.ts) | | Generated manifest | `docs-structure.json` | -| Generated Crowdin config | [crowdin.yml](crowdin.yml) | -| npm scripts | [package.json](package.json#L8-L11) | +| Generated Crowdin config | `crowdin.yml` | +| npm scripts | [package.json](package.json#L10-L11) |