diff --git a/README.md b/README.md index 02dd3d274f..7d6658b9fc 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,16 @@ signAddon({ }); ``` +## Reproducible builds + +`web-ext build` produces byte-for-byte identical ZIP archives when run repeatedly against the same source tree. Entries are added in a stable alphabetical order and every entry's timestamp is fixed, so the archive's checksum only changes when the source content changes. This is useful when [addons.mozilla.org](https://addons.mozilla.org) reviewers ask for a build script that reproduces the exact uploaded artifact. + +The fixed timestamp itself is arbitrary — what matters for reproducibility is that it doesn't depend on when the build runs. By default, every entry is stamped with `1980-01-01T00:00:00Z`, the earliest time the ZIP format can represent, picked because it makes it obvious the timestamp is synthetic rather than a real file mtime. To use a meaningful timestamp instead — for example, the commit time of the source tree — set the [`SOURCE_DATE_EPOCH`](https://reproducible-builds.org/docs/source-date-epoch/) environment variable to a Unix timestamp in seconds before invoking `web-ext build`: + +```sh +SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) web-ext build +``` + ## Should I Use It? Yes! The web-ext tool enables you to build and ship extensions for Firefox. diff --git a/package-lock.json b/package-lock.json index 504599626d..962f8c9299 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,8 +33,7 @@ "tmp": "0.2.5", "update-notifier": "7.3.1", "watchpack": "2.5.1", - "yargs": "17.7.2", - "zip-dir": "2.0.0" + "yargs": "17.7.2" }, "bin": { "web-ext": "bin/web-ext.js" @@ -153,6 +152,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2779,13 +2779,15 @@ "node_modules/@types/node": { "version": "20.5.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", - "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==" + "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", + "peer": true }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3412,11 +3414,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, "node_modules/async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -3697,6 +3694,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3871,6 +3869,7 @@ "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -5006,6 +5005,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.0.tgz", "integrity": "sha512-iy2GE3MHrYTL5lrCtMZ0X1KLEKKUjmK0kzwcnefhR66txcEmXZD2YWgR5GNdcEwkNx3a0siYkSvl0vIC+Svjmg==", "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -7124,6 +7124,7 @@ "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "devOptional": true, "license": "MIT", + "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -8937,6 +8938,7 @@ "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11295,15 +11297,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/zip-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/zip-dir/-/zip-dir-2.0.0.tgz", - "integrity": "sha512-uhlsJZWz26FLYXOD6WVuq+fIcZ3aBPGo/cFdiLlv3KNwpa52IF3ISV8fLhQLiqVu5No3VhlqlgthN6gehil1Dg==", - "dependencies": { - "async": "^3.2.0", - "jszip": "^3.2.2" - } } }, "dependencies": { @@ -11358,6 +11351,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, + "peer": true, "requires": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -13097,12 +13091,14 @@ "@types/node": { "version": "20.5.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", - "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==" + "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", + "peer": true }, "acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==" + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "peer": true }, "acorn-jsx": { "version": "5.3.2", @@ -13517,11 +13513,6 @@ "is-array-buffer": "^3.0.4" } }, - "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, "async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -13713,6 +13704,7 @@ "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", "dev": true, + "peer": true, "requires": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -13819,7 +13811,8 @@ "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", - "dev": true + "dev": true, + "peer": true }, "chai-as-promised": { "version": "8.0.2", @@ -14608,6 +14601,7 @@ "version": "9.39.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.0.tgz", "integrity": "sha512-iy2GE3MHrYTL5lrCtMZ0X1KLEKKUjmK0kzwcnefhR66txcEmXZD2YWgR5GNdcEwkNx3a0siYkSvl0vIC+Svjmg==", + "peer": true, "requires": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -16002,7 +15996,8 @@ "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "devOptional": true + "devOptional": true, + "peer": true }, "jose": { "version": "5.9.6", @@ -17311,7 +17306,8 @@ "version": "3.8.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", - "dev": true + "dev": true, + "peer": true }, "pretty-quick": { "version": "4.2.2", @@ -18938,15 +18934,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" - }, - "zip-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/zip-dir/-/zip-dir-2.0.0.tgz", - "integrity": "sha512-uhlsJZWz26FLYXOD6WVuq+fIcZ3aBPGo/cFdiLlv3KNwpa52IF3ISV8fLhQLiqVu5No3VhlqlgthN6gehil1Dg==", - "requires": { - "async": "^3.2.0", - "jszip": "^3.2.2" - } } } } diff --git a/package.json b/package.json index 381077e936..c7c539fa91 100644 --- a/package.json +++ b/package.json @@ -83,8 +83,7 @@ "tmp": "0.2.5", "update-notifier": "7.3.1", "watchpack": "2.5.1", - "yargs": "17.7.2", - "zip-dir": "2.0.0" + "yargs": "17.7.2" }, "devDependencies": { "@babel/cli": "7.28.6", diff --git a/src/cmd/build.js b/src/cmd/build.js index 37b6678d24..fc58f7d7f9 100644 --- a/src/cmd/build.js +++ b/src/cmd/build.js @@ -5,7 +5,7 @@ import fs from 'fs/promises'; import parseJSON from 'parse-json'; import stripBom from 'strip-bom'; import defaultFromEvent from 'promise-toolbox/fromEvent'; -import zipDir from 'zip-dir'; +import JSZip from 'jszip'; import defaultSourceWatcher from '../watcher.js'; import getValidatedManifest, { getManifestId } from '../util/manifest.js'; @@ -17,6 +17,77 @@ import { createFileFilter as defaultFileFilterCreator } from '../util/file-filte const log = createLogger(import.meta.url); const DEFAULT_FILENAME_TEMPLATE = '{name}-{version}.zip'; +// 1980-01-01 UTC — the earliest timestamp representable in a ZIP entry. +const ZIP_EPOCH_SECONDS = 315532800; + +// Build a ZIP buffer with deterministic byte output: entries are sorted +// alphabetically, and every entry's timestamp is fixed (overridable via the +// SOURCE_DATE_EPOCH environment variable, per the Reproducible Builds spec). +export async function createDeterministicZip(sourceDir, { filter } = {}) { + const epochSeconds = process.env.SOURCE_DATE_EPOCH + ? Number(process.env.SOURCE_DATE_EPOCH) + : ZIP_EPOCH_SECONDS; + if (!Number.isFinite(epochSeconds)) { + throw new UsageError( + `Invalid SOURCE_DATE_EPOCH value: ${process.env.SOURCE_DATE_EPOCH}`, + ); + } + const fixedDate = new Date(epochSeconds * 1000); + + const resolvedRoot = path.resolve(sourceDir); + const entries = []; + + async function walk(dir) { + const dirents = await fs.readdir(dir, { withFileTypes: true }); + dirents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const dirent of dirents) { + const full = path.join(dir, dirent.name); + const stat = await fs.lstat(full); + if (filter && !filter(full, stat)) { + continue; + } + if (dirent.isDirectory()) { + entries.push({ full, isDir: true }); + await walk(full); + } else if (dirent.isFile()) { + entries.push({ full, isDir: false }); + } + } + } + await walk(resolvedRoot); + + const zip = new JSZip(); + for (const entry of entries) { + // ZIP entries always use forward slashes, regardless of host OS. + const relative = path + .relative(resolvedRoot, entry.full) + .split(path.sep) + .join('/'); + // `createFolders: false` keeps JSZip from injecting parent-folder + // entries with a current-time `Date()`. The walk above already emits + // every directory explicitly, so we control all entry timestamps. + if (entry.isDir) { + zip.file(relative, null, { + dir: true, + date: fixedDate, + createFolders: false, + }); + } else { + const data = await fs.readFile(entry.full); + zip.file(relative, data, { date: fixedDate, createFolders: false }); + } + } + + // platform: 'UNIX' fixes the external-attribute byte; without it JSZip + // derives it from process.platform, which would make the same source + // produce different bytes on Windows vs Linux. + return zip.generateAsync({ + compression: 'DEFLATE', + type: 'nodebuffer', + platform: 'UNIX', + }); +} + export function safeFileName(name) { return name.toLowerCase().replace(/[^a-z0-9.-]+/g, '_'); } @@ -131,7 +202,7 @@ export async function defaultPackageCreator( manifestData = await getValidatedManifest(sourceDir); } - const buffer = await zipDir(sourceDir, { + const buffer = await createDeterministicZip(sourceDir, { filter: (...args) => fileFilter.wantFile(...args), }); diff --git a/tests/unit/test-cmd/test.build.js b/tests/unit/test-cmd/test.build.js index c4431eb966..00e2d02524 100644 --- a/tests/unit/test-cmd/test.build.js +++ b/tests/unit/test-cmd/test.build.js @@ -6,6 +6,7 @@ import { it, describe } from 'mocha'; import { assert } from 'chai'; import * as sinon from 'sinon'; import * as promiseToolbox from 'promise-toolbox'; +import JSZip from 'jszip'; import build, { safeFileName, @@ -58,6 +59,60 @@ describe('build', () => { ); }); + async function assertAllEntriesHaveDate(extensionPath, expectedMs) { + const zip = await JSZip.loadAsync(await fs.readFile(extensionPath)); + const entryNames = Object.keys(zip.files); + assert.isAbove( + entryNames.length, + 0, + 'zip should contain at least one entry', + ); + for (const name of entryNames) { + // ZIP DOS time encodes seconds in 5 bits (halved), so the on-disk + // resolution is 2 seconds. JSZip rounds when generating, so the + // round-tripped Date may be up to 2s off the value we passed in. + assert.closeTo( + zip.files[name].date.getTime(), + expectedMs, + 2000, + `entry ${name} has unexpected timestamp ${zip.files[name].date.toISOString()}`, + ); + } + } + + it('stamps every zip entry with the default fixed timestamp', () => + withTempDir(async (tmpDir) => { + const { extensionPath } = await build({ + sourceDir: fixturePath('minimal-web-ext'), + artifactsDir: tmpDir.path(), + }); + // Default is 1980-01-01T00:00:00Z, the earliest time the ZIP format + // can represent. The actual value doesn't matter — what matters is + // that it is fixed, not derived from "now". + await assertAllEntriesHaveDate(extensionPath, Date.UTC(1980, 0, 1)); + })); + + it('stamps every zip entry with SOURCE_DATE_EPOCH when set', async () => { + const epochSeconds = 1700000000; + const original = process.env.SOURCE_DATE_EPOCH; + process.env.SOURCE_DATE_EPOCH = String(epochSeconds); + try { + await withTempDir(async (tmpDir) => { + const { extensionPath } = await build({ + sourceDir: fixturePath('minimal-web-ext'), + artifactsDir: tmpDir.path(), + }); + await assertAllEntriesHaveDate(extensionPath, epochSeconds * 1000); + }); + } finally { + if (original === undefined) { + delete process.env.SOURCE_DATE_EPOCH; + } else { + process.env.SOURCE_DATE_EPOCH = original; + } + } + }); + it('configures a build command with the expected fileFilter', () => { const packageCreator = sinon.spy(() => ({ extensionPath: 'extension/path',