Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@

All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## Unreleased [patch]

> Development of this release was supported by [Reset Tech](https://www.reset.tech).

### Fixed

- Fix dataset publishing failure for large files by executing uploads sequentially instead of in parallel

## 10.3.2 - 2026-01-14

> Development of this release was supported by [Reset Tech](https://www.reset.tech).
Expand Down
2 changes: 1 addition & 1 deletion scripts/dataset/publish/github/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export default async function publish({ archivePath, releaseDate, stats }) {
logger.info('Uploading release asset…');

await octokit.rest.repos.uploadReleaseAsset({
data: fsApi.readFileSync(archivePath),
data: fsApi.createReadStream(archivePath),
headers: {
'content-type': 'application/zip',
'content-length': fsApi.statSync(archivePath).size,
Expand Down
29 changes: 16 additions & 13 deletions scripts/dataset/publish/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,29 @@ export default async function publishRelease({ archivePath, releaseDate, stats }
throw new Error('No publishing platform configured. Please configure at least one of: GitHub (OTA_ENGINE_GITHUB_TOKEN), GitLab (OTA_ENGINE_GITLAB_TOKEN), or data.gouv.fr (OTA_ENGINE_DATAGOUV_API_KEY + datasetId or organizationIdOrSlug in config).');
}

const results = await Promise.allSettled(platforms.map(async platform => {
const url = await platform.publish();

return { platform: platform.name, url };
}));

const succeeded = results.filter(result => result.status === 'fulfilled');
const failed = results.filter(result => result.status === 'rejected');
const succeeded = [];
const failed = [];

// Execute publications sequentially to avoid memory issues with large file uploads
for (const platform of platforms) {
try {
const url = await platform.publish();

succeeded.push({ platform: platform.name, url });
} catch (error) {
failed.push({ platform: platform.name, error });
}
}

if (failed.length) {
let errorMessage = !succeeded.length ? 'All platforms failed to publish:' : 'Some platforms failed to publish:';

failed.forEach(rejectedResult => {
const index = results.indexOf(rejectedResult);

errorMessage += `\n - ${platforms[index].name}: ${rejectedResult.reason.message}`;
failed.forEach(({ platform, error }) => {
errorMessage += `\n - ${platform}: ${error.message}`;
});

logger.error(errorMessage);
}

return succeeded.map(result => result.value);
return succeeded;
}