Skip to content

Fall back to .tar.gz sdists when no wheel is available #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
simonw merged 1 commit into from
Jun 9, 2026
+215 −29
Merged

Fall back to .tar.gz sdists when no wheel is available #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 122 additions & 23 deletions pypi-changelog.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@
<h1>PyPI Package Changelog</h1>
<input type="text" id="packageInput" placeholder="Package name (e.g. requests, flask, llm)" onkeydown="if(event.key==='Enter')lookupPackage()">
<button onclick="lookupPackage()">Look Up Package</button>
<p>Enter a PyPI package name to see all versions and compare changes between them.</p>
<p>Enter a PyPI package name to see all versions and compare changes between them. Wheels are used when available, with a fallback to the source distribution (<a href="?package=apple-fm-sdk">try apple-fm-sdk</a>).</p>
<div id="versionList" style="display: none"></div>

<script src="https://cdn.jsdelivr.net/npm/jszip@3.10.1/dist/jszip.min.js"></script>
Expand Down Expand Up @@ -238,29 +238,30 @@ <h1>PyPI Package Changelog</h1>
packageData = await response.json();
versionWheels = {};

// Get all versions with wheel URLs
// Get all versions that ship a wheel or a source distribution (.tar.gz)
const releases = packageData.releases;
const hasDist = files => files.some(f => f.filename.endsWith('.whl') || f.filename.endsWith('.tar.gz'));
const versions = Object.keys(releases)
.filter(v => {
const files = releases[v];
return files.some(f => f.filename.endsWith('.whl'));
})
.filter(v => hasDist(releases[v]))
.sort(compareVersions)
.reverse();

if (versions.length === 0) {
versionListEl.innerHTML = `<div class="status error">No wheel files found for "${escapeHtml(packageName)}"</div>`;
versionListEl.innerHTML = `<div class="status error">No wheel or source distribution files found for "${escapeHtml(packageName)}"</div>`;
return;
}

// Store wheel URLs for each version
// Store a dist URL for each version, preferring a wheel but falling
// back to a source distribution (.tar.gz) when no wheel is available
versions.forEach(v => {
const wheel = releases[v].find(f => f.filename.endsWith('.whl'));
if (wheel) {
const dist = wheel || releases[v].find(f => f.filename.endsWith('.tar.gz'));
if (dist) {
versionWheels[v] = {
url: wheel.url,
filename: wheel.filename,
uploadTime: wheel.upload_time_iso_8601
url: dist.url,
filename: dist.filename,
uploadTime: dist.upload_time_iso_8601,
isSdist: !wheel
};
}
});
Expand Down Expand Up @@ -297,7 +298,7 @@ <h1>PyPI Package Changelog</h1>

function renderVersionList(versions) {
const versionListEl = document.getElementById('versionList');
versionListEl.innerHTML = `<h3>${escapeHtml(packageData.info.name)} - ${versions.length} versions with wheels</h3>`;
versionListEl.innerHTML = `<h3>${escapeHtml(packageData.info.name)} - ${versions.length} versions</h3>`;

versions.forEach((version, index) => {
const info = versionWheels[version];
Expand Down Expand Up @@ -364,14 +365,28 @@ <h1>PyPI Package Changelog</h1>
return sample.indexOf('\0') !== -1;
}

function classifyContent(name, content) {
if (looksLikeBinary(content)) return '[Binary file]';
if (content.length > MAX_FILE_SIZE_FOR_DIFF) {
return '[Large file: ' + (content.length / 1024).toFixed(0) + 'KB]';
}
return content;
}

async function fetchWheelContents(version) {
const info = versionWheels[version];
if (!info) throw new Error(`No wheel found for version ${version}`);
if (!info) throw new Error(`No dist found for version ${version}`);

const response = await fetch(info.url);
const arrayBuffer = await response.arrayBuffer();
const zip = await JSZip.loadAsync(arrayBuffer);
const bytes = new Uint8Array(arrayBuffer);

// Source distributions are gzipped tarballs (gzip magic: 0x1f 0x8b)
if (info.isSdist || (bytes[0] === 0x1f && bytes[1] === 0x8b)) {
return fetchSdistContents(arrayBuffer);
}

const zip = await JSZip.loadAsync(arrayBuffer);
const files = {};
for (const [name, file] of Object.entries(zip.files)) {
if (!file.dir) {
Expand All @@ -380,14 +395,7 @@ <h1>PyPI Package Changelog</h1>
continue;
}
try {
const content = await file.async('text');
if (looksLikeBinary(content)) {
files[name] = '[Binary file]';
} else if (content.length > MAX_FILE_SIZE_FOR_DIFF) {
files[name] = '[Large file: ' + (content.length / 1024).toFixed(0) + 'KB]';
} else {
files[name] = content;
}
files[name] = classifyContent(name, await file.async('text'));
} catch (e) {
files[name] = '[Binary file]';
}
Expand All @@ -396,6 +404,97 @@ <h1>PyPI Package Changelog</h1>
return files;
}

async function fetchSdistContents(arrayBuffer) {
const tarBuffer = await gunzip(arrayBuffer);
const decoder = new TextDecoder('utf-8');
const files = {};
for (const entry of parseTar(tarBuffer)) {
// Strip the leading "<name>-<version>/" component so paths line up
// across versions when diffing source distributions
const name = entry.name.replace(/^[^/]+\//, '');
if (!name) continue;
if (isBinaryFilename(name)) {
files[name] = '[Binary file]';
continue;
}
try {
files[name] = classifyContent(name, decoder.decode(entry.data));
} catch (e) {
files[name] = '[Binary file]';
}
}
return files;
}

// Decompress gzip data using the browser's native DecompressionStream
async function gunzip(arrayBuffer) {
if (typeof DecompressionStream === 'undefined') {
throw new Error('Your browser does not support gzip decompression (DecompressionStream)');
}
const stream = new Response(arrayBuffer).body.pipeThrough(new DecompressionStream('gzip'));
return new Response(stream).arrayBuffer();
}

// Parse a tar archive (ArrayBuffer) into [{name, data}], handling ustar
// prefixes, GNU long names and PAX extended headers.
function parseTar(buffer) {
const bytes = new Uint8Array(buffer);
const decoder = new TextDecoder('utf-8');
const entries = [];
let offset = 0;
let longName = null; // GNU 'L' long name
let paxName = null; // PAX 'x' path override

const readStr = (start, len) => {
let end = start;
const max = start + len;
while (end < max && bytes[end] !== 0) end++;
return decoder.decode(bytes.subarray(start, end));
};

while (offset + 512 <= bytes.length) {
// End of archive is marked by a zero-filled block
let allZero = true;
for (let i = 0; i < 512; i++) {
if (bytes[offset + i] !== 0) { allZero = false; break; }
}
if (allZero) break;

let name = readStr(offset, 100);
const size = parseInt(readStr(offset + 124, 12).trim(), 8) || 0;
const typeflag = bytes[offset + 156];
const magic = readStr(offset + 257, 6);
const prefix = readStr(offset + 345, 155);
if (prefix && magic.startsWith('ustar')) {
name = prefix + '/' + name;
}

const dataStart = offset + 512;
offset = dataStart + Math.ceil(size / 512) * 512;

if (typeflag === 0x4c) { // 'L' GNU long name
longName = decoder.decode(bytes.subarray(dataStart, dataStart + size)).replace(/\0+$/, '');
continue;
}
if (typeflag === 0x78 || typeflag === 0x67) { // 'x' or 'g' PAX header
const records = decoder.decode(bytes.subarray(dataStart, dataStart + size));
const m = records.match(/^\d+ path=(.*)$/m);
if (m) paxName = m[1];
continue;
}

const effectiveName = paxName || longName || name;
paxName = null;
longName = null;

// Only keep regular files ('0' or legacy '\0'); skip directories etc.
if (typeflag === 0x30 || typeflag === 0) {
entries.push({ name: effectiveName, data: bytes.subarray(dataStart, dataStart + size) });
}
}
return entries;
}

function matchFiles(oldFiles, newFiles) {
// Returns array of {oldName, newName, oldContent, newContent}
// Handles renames for .dist-info files
Expand Down
99 changes: 93 additions & 6 deletions zip-wheel-explorer.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,9 @@
</head>
<body>
<h1>Package File Browser</h1>
<input type="text" id="urlInput" placeholder="Package name (e.g. requests) or URL to .whl/.zip file" onkeydown="if(event.key==='Enter')fetchAndUnpackFile()">
<input type="text" id="urlInput" placeholder="Package name (e.g. requests) or URL to .whl/.zip/.tar.gz file" onkeydown="if(event.key==='Enter')fetchAndUnpackFile()">
<button onclick="fetchAndUnpackFile()">Unpack File</button>
<p>Enter a PyPI package name like <code>requests</code> or <code>llm-mistral</code> (<a href="?package=llm-mistral">try it</a>), or paste a direct URL to a .whl or .zip file.</p>
<p>Enter a PyPI package name like <code>requests</code> or <code>llm-mistral</code> (<a href="?package=llm-mistral">try it</a>), or paste a direct URL to a .whl, .zip or .tar.gz file. Packages with no wheel fall back to the source distribution (<a href="?package=apple-fm-sdk">try apple-fm-sdk</a>).</p>
<div id="fileList" style="display: none"></div>

<script src="https://cdn.jsdelivr.net/npm/jszip@3.10.1/dist/jszip.min.js"></script>
Expand Down Expand Up @@ -336,12 +336,14 @@ <h1>Package File Browser</h1>
return;
}
const pypiData = await pypiResponse.json();
const wheelUrl = pypiData.urls?.find(u => u.url?.endsWith('.whl'));
if (!wheelUrl) {
fileListEl.innerHTML = `<div class="status">No wheel (.whl) found for "${escapeHtml(input)}"</div>`;
// Prefer a wheel, but fall back to a source distribution (.tar.gz)
const fileUrl = pypiData.urls?.find(u => u.url?.endsWith('.whl'))
|| pypiData.urls?.find(u => u.url?.endsWith('.tar.gz'));
if (!fileUrl) {
fileListEl.innerHTML = `<div class="status">No wheel (.whl) or source distribution (.tar.gz) found for "${escapeHtml(input)}"</div>`;
return;
}
url = wheelUrl.url;
url = fileUrl.url;
} catch (error) {
fileListEl.innerHTML = `<div class="status">Error looking up package: ${escapeHtml(error.message)}</div>`;
return;
Expand Down Expand Up @@ -396,6 +398,12 @@ <h1>Package File Browser</h1>
}

async function extractFiles(arrayBuffer, url) {
const bytes = new Uint8Array(arrayBuffer);
// Gzip magic bytes (0x1f 0x8b) indicate a .tar.gz source distribution
if (bytes[0] === 0x1f && bytes[1] === 0x8b) {
return extractTarGz(arrayBuffer);
}
// Otherwise treat as a zip archive (.whl / .zip)
const files = [];
const zip = await JSZip.loadAsync(arrayBuffer);
for (const [name, file] of Object.entries(zip.files)) {
Expand All @@ -406,6 +414,85 @@ <h1>Package File Browser</h1>
return files;
}

// Decompress gzip data using the browser's native DecompressionStream
async function gunzip(arrayBuffer) {
if (typeof DecompressionStream === 'undefined') {
throw new Error('Your browser does not support gzip decompression (DecompressionStream)');
}
const stream = new Response(arrayBuffer).body.pipeThrough(new DecompressionStream('gzip'));
return new Response(stream).arrayBuffer();
}

// Parse a tar archive (Uint8Array) into [{name, data}], handling ustar
// prefixes, GNU long names and PAX extended headers.
function parseTar(buffer) {
const bytes = new Uint8Array(buffer);
const decoder = new TextDecoder('utf-8');
const entries = [];
let offset = 0;
let longName = null; // GNU 'L' long name
let paxName = null; // PAX 'x' path override

const readStr = (start, len) => {
let end = start;
const max = start + len;
while (end < max && bytes[end] !== 0) end++;
return decoder.decode(bytes.subarray(start, end));
};

while (offset + 512 <= bytes.length) {
// End of archive is marked by a zero-filled block
let allZero = true;
for (let i = 0; i < 512; i++) {
if (bytes[offset + i] !== 0) { allZero = false; break; }
}
if (allZero) break;

let name = readStr(offset, 100);
const size = parseInt(readStr(offset + 124, 12).trim(), 8) || 0;
const typeflag = bytes[offset + 156];
const magic = readStr(offset + 257, 6);
const prefix = readStr(offset + 345, 155);
if (prefix && magic.startsWith('ustar')) {
name = prefix + '/' + name;
}

const dataStart = offset + 512;
offset = dataStart + Math.ceil(size / 512) * 512;

if (typeflag === 0x4c) { // 'L' GNU long name
longName = decoder.decode(bytes.subarray(dataStart, dataStart + size)).replace(/\0+$/, '');
continue;
}
if (typeflag === 0x78 || typeflag === 0x67) { // 'x' or 'g' PAX header
const records = decoder.decode(bytes.subarray(dataStart, dataStart + size));
const m = records.match(/^\d+ path=(.*)$/m);
if (m) paxName = m[1];
continue;
}

const effectiveName = paxName || longName || name;
paxName = null;
longName = null;

// Only keep regular files ('0' or legacy '\0'); skip directories etc.
if (typeflag === 0x30 || typeflag === 0) {
entries.push({ name: effectiveName, data: bytes.subarray(dataStart, dataStart + size) });
}
}
return entries;
}

async function extractTarGz(arrayBuffer) {
const tarBuffer = await gunzip(arrayBuffer);
const decoder = new TextDecoder('utf-8');
return parseTar(tarBuffer).map(entry => ({
name: entry.name,
content: decoder.decode(entry.data),
size: entry.data.byteLength
}));
}

// Check for query params on page load
const params = new URLSearchParams(window.location.search);
const packageParam = params.get('package');
Expand Down
Loading