From cdee5133ac0d3764503a964f90dd67c4c6212e15 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 11 Feb 2026 12:43:11 -0300 Subject: [PATCH 1/4] Add `variant` field and NoGPL build support Add support for cataloging Julia's "nogpl" build variants (builds without GPL-licensed dependencies) hosted on the julialang-nogpl S3 bucket. - Add `variant` field ("default"/"nogpl") to schema and file metadata - Add `NoGPL{T}` parametric wrapper type with dispatch methods for URL generation matching the nogpl bucket structure - Add `get_tag_commits()` to resolve version tags to git commit hashes (nogpl filenames use short commit hashes instead of version numbers) - Add nogpl platform loop in `main()` covering Linux x64, macOS x64/aarch64, and Windows x64 Co-Authored-By: Claude Opus 4.6 --- schema.json | 12 +++ src/VersionsJSONUtil.jl | 168 ++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 30 ++++++- 3 files changed, 209 insertions(+), 1 deletion(-) diff --git a/schema.json b/schema.json index ffb1ad0..93f33fc 100644 --- a/schema.json +++ b/schema.json @@ -68,6 +68,9 @@ }, "extension": { "$ref": "#/definitions/FileExtension" + }, + "variant": { + "$ref": "#/definitions/Variant" } }, "required": [ @@ -79,6 +82,7 @@ "size", "triplet", "url", + "variant", "version" ], "title": "File" @@ -138,6 +142,14 @@ "zip" ], "title": "FileExtension" + }, + "Variant": { + "type": "string", + "enum": [ + "default", + "nogpl" + ], + "title": "Variant" } } } diff --git a/src/VersionsJSONUtil.jl b/src/VersionsJSONUtil.jl index 837d348..1d54bab 100644 --- a/src/VersionsJSONUtil.jl +++ b/src/VersionsJSONUtil.jl @@ -23,6 +23,12 @@ end MacOSTarball(arch::Symbol) = MacOSTarball(MacOS(arch)) @forward MacOSTarball.macos (up_os, tar_os, triplet, arch) +"Wrapper type for NoGPL build variants" +struct NoGPL{T} + platform::T +end +@forward NoGPL.platform (triplet, arch, meta_os, jlext) + up_os(p::Windows) = "winnt" up_os(p::MacOS) = "mac" up_os(p::Linux) = libc(p) == :glibc ? "linux" : "musl" @@ -65,6 +71,37 @@ jlext(p::WindowsTarball) = "tar.gz" jlext(p::MacOS) = "dmg" jlext(p) = "tar.gz" +# NoGPL-specific methods +function up_os(p::NoGPL) + base = up_os(p.platform) + base == "winnt" && return "windowsnogpl" + base == "mac" && return "macosnogpl" + return base * "nogpl" +end +up_arch(p::NoGPL) = up_arch(p.platform) + +function tar_os(p::NoGPL) + a = arch(p.platform) + if p.platform isa Windows || p.platform isa WindowsPortable || p.platform isa WindowsTarball + os_name = "windowsnogpl" + elseif p.platform isa MacOS || p.platform isa MacOSTarball + os_name = "macosnogpl" + elseif p.platform isa Linux + os_name = "linuxnogpl" + else + error("Unsupported NoGPL platform: $(p.platform)") + end + if a == :x86_64 + return "$(os_name)64" + else + return "$(os_name)-$(a)" + end +end + +# Variant name for file metadata +variant_name(p) = "default" +variant_name(p::NoGPL) = "nogpl" + # OS to use in the metadata # The OS in the download URL for Linux with musl is "musl" # But the OS in the metadata should be "linux" @@ -81,6 +118,16 @@ function download_url(version::VersionNumber, platform) ) end +function download_url(version::VersionNumber, platform::NoGPL, commit_short_hash::AbstractString) + return string( + "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/", + up_os(platform), "/", + up_arch(platform), "/", + version.major, ".", version.minor, "/", + "julia-", commit_short_hash, "-", tar_os(platform), ".", jlext(platform), + ) +end + # We're going to collect the combinatorial explosion of version/os-arch possible downloads. # We don't have a nice, neat list of what is or is not available, and so we're just going to # try and download each file, and if it exists, yay. Otherwise, bleh. @@ -109,6 +156,17 @@ julia_platforms = [ FreeBSD(:x86_64), ] +nogpl_platforms = [ + NoGPL(Linux(:x86_64; libc = :glibc)), + NoGPL(MacOS(:x86_64)), + NoGPL(MacOS(:aarch64)), + NoGPL(MacOSTarball(:x86_64)), + NoGPL(MacOSTarball(:aarch64)), + NoGPL(Windows(:x86_64)), + NoGPL(WindowsPortable(:x86_64)), + NoGPL(WindowsTarball(:x86_64)), +] + function vnum_maybe(x::AbstractString) try return VersionNumber(x) @@ -132,9 +190,33 @@ function get_tags() JSON.parse(String(read(tags_json_path))) end +# Get mapping of version → 10-char commit short hash using GitHub tags API +function get_tag_commits() + @info("Fetching tag→commit mappings...") + tag_commits = Dict{VersionNumber, String}() + page = 1 + while true + url = "https://api.github.com/repos/JuliaLang/julia/tags?per_page=100&page=$(page)" + cache_name = "julia_tags_page_$(page).json" + path = WebCacheUtilities.download_to_cache(cache_name, url) + tags = JSON.parse(String(read(path))) + isempty(tags) && break + for tag in tags + name = tag["name"] + v = vnum_maybe(name) + v === nothing && continue + sha = tag["commit"]["sha"] + tag_commits[v] = sha[1:10] + end + page += 1 + end + return tag_commits +end + function main(out_path) tags = get_tags() tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) + tag_commits = get_tag_commits() meta = Dict() number_urls_tried = 0 @@ -216,6 +298,7 @@ function main(out_path) "os" => meta_os(platform), "arch" => string(arch(platform)), "version" => string(version), + "variant" => variant_name(platform), "sha256" => tarball_hash, "size" => filesize(filepath), "kind" => kind, @@ -240,6 +323,91 @@ function main(out_path) rm(filepath) end end + # NoGPL variants + for version in tag_versions + commit_hash = get(tag_commits, version, nothing) + if commit_hash === nothing + @info "No commit hash found for $(version), skipping nogpl builds" + continue + end + for platform in nogpl_platforms + url = download_url(version, platform, commit_hash) + filename = basename(url) + + # Download this URL to a local file + number_urls_tried += 1 + local filepath + try + print(stdout, "Downloading $(filename)...") + filepath = WebCacheUtilities.download_to_cache(filename, url) + catch ex + if isa(ex, InterruptException) + rethrow(ex) + end + println(stdout, " ✗") + continue + end + number_urls_success += 1 + println(stdout, " ✓") + + tarball_hash_path = hit_file_cache("$(filename).sha256") do tarball_hash_path + open(filepath, "r") do io + open(tarball_hash_path, "w") do hash_io + write(hash_io, bytes2hex(sha256(io))) + end + end + end + tarball_hash = String(read(tarball_hash_path)) + + # Initialize overall version key, if needed + if !haskey(meta, version) + meta[version] = Dict( + "stable" => is_stable(version), + "files" => Vector{Dict}(), + ) + end + + # Build up metadata about this file + if endswith(filename, ".dmg") + kind = "archive" + extension = "dmg" + elseif endswith(filename, ".exe") + kind = "installer" + extension = "exe" + elseif endswith(filename, ".tar.gz") + kind = "archive" + extension = "tar.gz" + elseif endswith(filename, ".zip") + kind = "archive" + extension = "zip" + else + error("Unsupported file extension in filename: $(filename)") + end + file_dict = Dict( + "triplet" => triplet(platform), + "os" => meta_os(platform), + "arch" => string(arch(platform)), + "version" => string(version), + "variant" => variant_name(platform), + "sha256" => tarball_hash, + "size" => filesize(filepath), + "kind" => kind, + "extension" => extension, + "url" => url, + ) + + push!(meta[version]["files"], file_dict) + + # Write out new versions of our versions.json as we go + open(out_path, "w") do io + JSON.print(io, meta, 2) + end + + # Delete downloaded file + rm(filepath) + end + end + @info "Tried $(number_urls_tried) versions, successfully downloaded $(number_urls_success)" end diff --git a/test/runtests.jl b/test/runtests.jl index bf3186e..9a294f1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ using Pkg.BinaryPlatforms, JSON using VersionsJSONUtil -import VersionsJSONUtil: WindowsPortable, WindowsTarball, MacOSTarball +import VersionsJSONUtil: WindowsPortable, WindowsTarball, MacOSTarball, NoGPL using Test const download_urls = Dict( @@ -26,10 +26,38 @@ const download_urls = Dict( ), ) +const nogpl_download_urls = Dict( + v"1.10.0" => Dict( + NoGPL(Linux(:x86_64; libc = :glibc)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/linuxnogpl/x64/1.10/julia-abcdef0123-linuxnogpl64.tar.gz", + NoGPL(MacOS(:x86_64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/macosnogpl/x64/1.10/julia-abcdef0123-macosnogpl64.dmg", + NoGPL(MacOS(:aarch64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/macosnogpl/aarch64/1.10/julia-abcdef0123-macosnogpl-aarch64.dmg", + NoGPL(MacOSTarball(:x86_64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/macosnogpl/x64/1.10/julia-abcdef0123-macosnogpl64.tar.gz", + NoGPL(MacOSTarball(:aarch64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/macosnogpl/aarch64/1.10/julia-abcdef0123-macosnogpl-aarch64.tar.gz", + NoGPL(Windows(:x86_64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/windowsnogpl/x64/1.10/julia-abcdef0123-windowsnogpl64.exe", + NoGPL(WindowsPortable(:x86_64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/windowsnogpl/x64/1.10/julia-abcdef0123-windowsnogpl64.zip", + NoGPL(WindowsTarball(:x86_64)) => "https://julialang-nogpl.s3.amazonaws.com/bin-nogpl/windowsnogpl/x64/1.10/julia-abcdef0123-windowsnogpl64.tar.gz", + ), +) + @testset "VersionsJSONUtil.jl" begin @testset "Download URLs for $v" for v in keys(download_urls) for (p, url) in download_urls[v] @test VersionsJSONUtil.download_url(v, p) == url end end + + @testset "NoGPL Download URLs for $v" for v in keys(nogpl_download_urls) + for (p, url) in nogpl_download_urls[v] + @test VersionsJSONUtil.download_url(v, p, "abcdef0123") == url + end + end + + @testset "variant_name" begin + @test VersionsJSONUtil.variant_name(Linux(:x86_64)) == "default" + @test VersionsJSONUtil.variant_name(MacOS(:x86_64)) == "default" + @test VersionsJSONUtil.variant_name(Windows(:x86_64)) == "default" + @test VersionsJSONUtil.variant_name(NoGPL(Linux(:x86_64; libc = :glibc))) == "nogpl" + @test VersionsJSONUtil.variant_name(NoGPL(MacOS(:x86_64))) == "nogpl" + @test VersionsJSONUtil.variant_name(NoGPL(Windows(:x86_64))) == "nogpl" + end end From ead7ede3fc435e8aea36648c7111d2728ca55b61 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 11 Feb 2026 20:00:33 +0000 Subject: [PATCH 2/4] Refactor: extract process_download! and fix JSON key serialization - Extract shared download/hash/metadata logic into `process_download!` helper to eliminate ~80 lines of duplication between default and nogpl platform loops - Add .asc signature checking for nogpl builds (the S3 bucket has .asc files for nogpl archives) - Use string(version) as meta dict key to fix compatibility with newer JSON.jl which requires explicit key serialization Verified: output matches upstream versions.json with 0 mismatches across 2,147 files and 190 versions. Co-Authored-By: Claude Opus 4.6 --- src/VersionsJSONUtil.jl | 276 ++++++++++++++++------------------------ 1 file changed, 110 insertions(+), 166 deletions(-) diff --git a/src/VersionsJSONUtil.jl b/src/VersionsJSONUtil.jl index 1d54bab..fe3609e 100644 --- a/src/VersionsJSONUtil.jl +++ b/src/VersionsJSONUtil.jl @@ -213,6 +213,109 @@ function get_tag_commits() return tag_commits end +# Determine whether to check for an .asc signature for a given platform. +# Installers (MacOS .dmg and Windows .exe) don't have .asc files; +# archives (.tar.gz, .zip) do. +function _has_asc(platform) + inner = platform isa NoGPL ? platform.platform : platform + return !isa(inner, MacOS) && !isa(inner, Windows) +end + +# Download a file, compute its hash, check for .asc signature, build metadata, +# and write the updated versions.json. Returns true on success. +function process_download!(meta, out_path, url, platform, version) + filename = basename(url) + + # Download this URL to a local file + local filepath + try + print(stdout, "Downloading $(filename)...") + filepath = WebCacheUtilities.download_to_cache(filename, url) + catch ex + isa(ex, InterruptException) && rethrow(ex) + println(stdout, " ✗") + return false + end + println(stdout, " ✓") + + tarball_hash_path = hit_file_cache("$(filename).sha256") do tarball_hash_path + open(filepath, "r") do io + open(tarball_hash_path, "w") do hash_io + write(hash_io, bytes2hex(sha256(io))) + end + end + end + tarball_hash = String(read(tarball_hash_path)) + + # Initialize overall version key, if needed + vstr = string(version) + if !haskey(meta, vstr) + meta[vstr] = Dict( + "stable" => is_stable(version), + "files" => Vector{Dict}(), + ) + end + + # Test to see if there is an asc signature: + asc_signature = nothing + if _has_asc(platform) + asc_url = string(url, ".asc") + print(stdout, " Downloading $(basename(asc_url))") + try + asc_filepath = WebCacheUtilities.download_to_cache(basename(asc_url), asc_url) + asc_signature = String(read(asc_filepath)) + println(stdout, " ✓") + catch ex + isa(ex, InterruptException) && rethrow(ex) + println(stdout, " ✗") + end + end + + # Build up metadata about this file + if endswith(filename, ".dmg") + kind = "archive" + extension = "dmg" + elseif endswith(filename, ".exe") + kind = "installer" + extension = "exe" + elseif endswith(filename, ".tar.gz") + kind = "archive" + extension = "tar.gz" + elseif endswith(filename, ".zip") + kind = "archive" + extension = "zip" + else + error("Unsupported file extension in filename: $(filename)") + end + file_dict = Dict( + "triplet" => triplet(platform), + "os" => meta_os(platform), + "arch" => string(arch(platform)), + "version" => string(version), + "variant" => variant_name(platform), + "sha256" => tarball_hash, + "size" => filesize(filepath), + "kind" => kind, + "extension" => extension, + "url" => url, + ) + # Add in `.asc` signature content, if applicable + if asc_signature !== nothing + file_dict["asc"] = asc_signature + end + + push!(meta[vstr]["files"], file_dict) + + # Write out new versions of our versions.json as we go + open(out_path, "w") do io + JSON.print(io, meta, 2) + end + + # Delete downloaded file + rm(filepath) + return true +end + function main(out_path) tags = get_tags() tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) @@ -221,108 +324,18 @@ function main(out_path) meta = Dict() number_urls_tried = 0 number_urls_success = 0 + + # Default variants for version in tag_versions for platform in julia_platforms url = download_url(version, platform) - filename = basename(url) - - # Download this URL to a local file number_urls_tried += 1 - local filepath - try - print(stdout, "Downloading $(filename)...") - filepath = WebCacheUtilities.download_to_cache(filename, url) - catch ex - if isa(ex, InterruptException) - rethrow(ex) - end - println(stdout, " ✗") - continue - end - number_urls_success += 1 - println(stdout, " ✓") - - tarball_hash_path = hit_file_cache("$(filename).sha256") do tarball_hash_path - open(filepath, "r") do io - open(tarball_hash_path, "w") do hash_io - write(hash_io, bytes2hex(sha256(io))) - end - end - end - tarball_hash = String(read(tarball_hash_path)) - - # Initialize overall version key, if needed - if !haskey(meta, version) - meta[version] = Dict( - "stable" => is_stable(version), - "files" => Vector{Dict}(), - ) - end - - # Test to see if there is an asc signature: - asc_signature = nothing - if !isa(platform, MacOS) && !isa(platform, Windows) - asc_url = string(url, ".asc") - print(stdout, " Downloading $(basename(asc_url))") - try - asc_filepath = WebCacheUtilities.download_to_cache(basename(asc_url), asc_url) - asc_signature = String(read(asc_filepath)) - println(stdout, " ✓") - catch ex - if isa(ex, InterruptException) - rethrow(ex) - end - println(stdout, " ✗") - end - - end - - # Build up metadata about this file - if endswith(filename, ".dmg") - kind = "archive" - extension = "dmg" - elseif endswith(filename, ".exe") - kind = "installer" - extension = "exe" - elseif endswith(filename, ".tar.gz") - kind = "archive" - extension = "tar.gz" - elseif endswith(filename, ".zip") - kind = "archive" - extension = "zip" - else - error("Unsupported file extension in filename: $(filename)") - end - file_dict = Dict( - "triplet" => triplet(platform), - "os" => meta_os(platform), - "arch" => string(arch(platform)), - "version" => string(version), - "variant" => variant_name(platform), - "sha256" => tarball_hash, - "size" => filesize(filepath), - "kind" => kind, - "extension" => extension, - "url" => url, - ) - # Add in `.asc` signature content, if applicable - if asc_signature !== nothing - file_dict["asc"] = asc_signature - end - - # Right now, all we have are archives, but let's be forward-thinking - # and make this an array of dictionaries that is easy to extensibly match - push!(meta[version]["files"], file_dict) - - # Write out new versions of our versions.json as we go - open(out_path, "w") do io - JSON.print(io, meta, 2) + if process_download!(meta, out_path, url, platform, version) + number_urls_success += 1 end - - # Delete downloaded file - rm(filepath) end end + # NoGPL variants for version in tag_versions commit_hash = get(tag_commits, version, nothing) @@ -332,79 +345,10 @@ function main(out_path) end for platform in nogpl_platforms url = download_url(version, platform, commit_hash) - filename = basename(url) - - # Download this URL to a local file number_urls_tried += 1 - local filepath - try - print(stdout, "Downloading $(filename)...") - filepath = WebCacheUtilities.download_to_cache(filename, url) - catch ex - if isa(ex, InterruptException) - rethrow(ex) - end - println(stdout, " ✗") - continue + if process_download!(meta, out_path, url, platform, version) + number_urls_success += 1 end - number_urls_success += 1 - println(stdout, " ✓") - - tarball_hash_path = hit_file_cache("$(filename).sha256") do tarball_hash_path - open(filepath, "r") do io - open(tarball_hash_path, "w") do hash_io - write(hash_io, bytes2hex(sha256(io))) - end - end - end - tarball_hash = String(read(tarball_hash_path)) - - # Initialize overall version key, if needed - if !haskey(meta, version) - meta[version] = Dict( - "stable" => is_stable(version), - "files" => Vector{Dict}(), - ) - end - - # Build up metadata about this file - if endswith(filename, ".dmg") - kind = "archive" - extension = "dmg" - elseif endswith(filename, ".exe") - kind = "installer" - extension = "exe" - elseif endswith(filename, ".tar.gz") - kind = "archive" - extension = "tar.gz" - elseif endswith(filename, ".zip") - kind = "archive" - extension = "zip" - else - error("Unsupported file extension in filename: $(filename)") - end - file_dict = Dict( - "triplet" => triplet(platform), - "os" => meta_os(platform), - "arch" => string(arch(platform)), - "version" => string(version), - "variant" => variant_name(platform), - "sha256" => tarball_hash, - "size" => filesize(filepath), - "kind" => kind, - "extension" => extension, - "url" => url, - ) - - push!(meta[version]["files"], file_dict) - - # Write out new versions of our versions.json as we go - open(out_path, "w") do io - JSON.print(io, meta, 2) - end - - # Delete downloaded file - rm(filepath) end end From e6e9fdbe1dc6ff610460493933d0499f920d186e Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 12 Feb 2026 11:50:38 +0000 Subject: [PATCH 3/4] Separate nogpl into its own versions.json per bucket Instead of adding a variant field to the shared schema, generate separate versions files per bucket: - julialang-s3: versions.json (unchanged schema, no variant field) - julialang-nogpl: versions.v1.json (same schema, separate file) This keeps existing consumers unaffected. The process_download! refactor is retained to share download/hash/metadata logic between main() and main_nogpl(). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/CI.yml | 28 +++++++++++++++++++++++++++- schema.json | 12 ------------ src/VersionsJSONUtil.jl | 20 +++++++++++--------- test/runtests.jl | 11 ++++------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 57244b8..be5ad87 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,6 +23,7 @@ permissions: env: aws_region: us-east-1 s3_bucket: julialang2 + s3_bucket_nogpl: julialang-nogpl jobs: package-tests: @@ -73,7 +74,7 @@ jobs: - name: Install dependencies run: julia --color=yes --project -e "using Pkg; Pkg.instantiate()" - - run: rm -f versions.json + - run: rm -f versions.json versions-nogpl.json - name: Build versions.json run: | @@ -82,9 +83,19 @@ jobs: VersionsJSONUtil.main("versions.json") shell: julia --project {0} + - name: Build versions-nogpl.json + run: | + using VersionsJSONUtil + + VersionsJSONUtil.main_nogpl("versions-nogpl.json") + shell: julia --project {0} + - name: Validate versions.json against schema run: npx -p ajv-cli@3.3.0 ajv -s schema.json -d versions.json + - name: Validate versions-nogpl.json against schema + run: npx -p ajv-cli@3.3.0 ajv -s schema.json -d versions-nogpl.json + - run: julia --project test/more_tests.jl versions.json - name: Upload versions.json as workflow artifact @@ -94,6 +105,13 @@ jobs: path: versions.json if-no-files-found: error + - name: Upload versions-nogpl.json as workflow artifact + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: versions-nogpl + path: versions-nogpl.json + if-no-files-found: error + upload-to-s3: needs: [package-tests, full-test] if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' @@ -106,6 +124,11 @@ jobs: with: name: versions + - name: Download versions-nogpl.json from previous job + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + name: versions-nogpl + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 with: @@ -116,5 +139,8 @@ jobs: - name: Upload versions.json to S3 run: aws s3 cp versions.json s3://${{ env.s3_bucket }}/bin/versions.json --acl public-read --no-progress + - name: Upload versions-nogpl.json to S3 + run: aws s3 cp versions-nogpl.json s3://${{ env.s3_bucket_nogpl }}/bin-nogpl/versions.v1.json --acl public-read --no-progress + - name: Purge cache run: curl -X PURGE https://julialang-s3.julialang.org/bin/versions.json diff --git a/schema.json b/schema.json index 93f33fc..ffb1ad0 100644 --- a/schema.json +++ b/schema.json @@ -68,9 +68,6 @@ }, "extension": { "$ref": "#/definitions/FileExtension" - }, - "variant": { - "$ref": "#/definitions/Variant" } }, "required": [ @@ -82,7 +79,6 @@ "size", "triplet", "url", - "variant", "version" ], "title": "File" @@ -142,14 +138,6 @@ "zip" ], "title": "FileExtension" - }, - "Variant": { - "type": "string", - "enum": [ - "default", - "nogpl" - ], - "title": "Variant" } } } diff --git a/src/VersionsJSONUtil.jl b/src/VersionsJSONUtil.jl index fe3609e..0ec738d 100644 --- a/src/VersionsJSONUtil.jl +++ b/src/VersionsJSONUtil.jl @@ -98,10 +98,6 @@ function tar_os(p::NoGPL) end end -# Variant name for file metadata -variant_name(p) = "default" -variant_name(p::NoGPL) = "nogpl" - # OS to use in the metadata # The OS in the download URL for Linux with musl is "musl" # But the OS in the metadata should be "linux" @@ -292,7 +288,6 @@ function process_download!(meta, out_path, url, platform, version) "os" => meta_os(platform), "arch" => string(arch(platform)), "version" => string(version), - "variant" => variant_name(platform), "sha256" => tarball_hash, "size" => filesize(filepath), "kind" => kind, @@ -319,13 +314,10 @@ end function main(out_path) tags = get_tags() tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) - tag_commits = get_tag_commits() meta = Dict() number_urls_tried = 0 number_urls_success = 0 - - # Default variants for version in tag_versions for platform in julia_platforms url = download_url(version, platform) @@ -336,7 +328,17 @@ function main(out_path) end end - # NoGPL variants + @info "Tried $(number_urls_tried) versions, successfully downloaded $(number_urls_success)" +end + +function main_nogpl(out_path) + tags = get_tags() + tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) + tag_commits = get_tag_commits() + + meta = Dict() + number_urls_tried = 0 + number_urls_success = 0 for version in tag_versions commit_hash = get(tag_commits, version, nothing) if commit_hash === nothing diff --git a/test/runtests.jl b/test/runtests.jl index 9a294f1..0c91077 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -52,12 +52,9 @@ const nogpl_download_urls = Dict( end end - @testset "variant_name" begin - @test VersionsJSONUtil.variant_name(Linux(:x86_64)) == "default" - @test VersionsJSONUtil.variant_name(MacOS(:x86_64)) == "default" - @test VersionsJSONUtil.variant_name(Windows(:x86_64)) == "default" - @test VersionsJSONUtil.variant_name(NoGPL(Linux(:x86_64; libc = :glibc))) == "nogpl" - @test VersionsJSONUtil.variant_name(NoGPL(MacOS(:x86_64))) == "nogpl" - @test VersionsJSONUtil.variant_name(NoGPL(Windows(:x86_64))) == "nogpl" + @testset "meta_os for NoGPL" begin + @test VersionsJSONUtil.meta_os(NoGPL(Linux(:x86_64; libc = :glibc))) == "linux" + @test VersionsJSONUtil.meta_os(NoGPL(MacOS(:x86_64))) == "mac" + @test VersionsJSONUtil.meta_os(NoGPL(Windows(:x86_64))) == "winnt" end end From d8b7941d5c915b20f05f828205dc907877e4531f Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 12 Feb 2026 12:05:35 +0000 Subject: [PATCH 4/4] Separate nogpl into its own versions.json per bucket Instead of adding a variant field to the shared schema, generate separate versions files per bucket: - julialang-s3: versions.json (unchanged schema) - julialang-nogpl: versions.v1.json (same schema, separate file) Extract process_download! and _build_versions helpers to share download/hash/metadata logic between main() and main_nogpl(). Co-Authored-By: Claude Opus 4.6 --- src/VersionsJSONUtil.jl | 110 +++++++++++++++------------------------- 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/src/VersionsJSONUtil.jl b/src/VersionsJSONUtil.jl index 0ec738d..46f8060 100644 --- a/src/VersionsJSONUtil.jl +++ b/src/VersionsJSONUtil.jl @@ -71,31 +71,21 @@ jlext(p::WindowsTarball) = "tar.gz" jlext(p::MacOS) = "dmg" jlext(p) = "tar.gz" -# NoGPL-specific methods -function up_os(p::NoGPL) - base = up_os(p.platform) - base == "winnt" && return "windowsnogpl" - base == "mac" && return "macosnogpl" - return base * "nogpl" +# NoGPL-specific methods: the S3 bucket uses a uniform {os}nogpl naming scheme +function _nogpl_os(p) + (p isa Windows || p isa WindowsPortable || p isa WindowsTarball) && return "windowsnogpl" + (p isa MacOS || p isa MacOSTarball) && return "macosnogpl" + p isa Linux && return "linuxnogpl" + error("Unsupported NoGPL platform: $p") end + +up_os(p::NoGPL) = _nogpl_os(p.platform) up_arch(p::NoGPL) = up_arch(p.platform) function tar_os(p::NoGPL) + os = _nogpl_os(p.platform) a = arch(p.platform) - if p.platform isa Windows || p.platform isa WindowsPortable || p.platform isa WindowsTarball - os_name = "windowsnogpl" - elseif p.platform isa MacOS || p.platform isa MacOSTarball - os_name = "macosnogpl" - elseif p.platform isa Linux - os_name = "linuxnogpl" - else - error("Unsupported NoGPL platform: $(p.platform)") - end - if a == :x86_64 - return "$(os_name)64" - else - return "$(os_name)-$(a)" - end + return a == :x86_64 ? "$(os)64" : "$(os)-$(a)" end # OS to use in the metadata @@ -209,14 +199,6 @@ function get_tag_commits() return tag_commits end -# Determine whether to check for an .asc signature for a given platform. -# Installers (MacOS .dmg and Windows .exe) don't have .asc files; -# archives (.tar.gz, .zip) do. -function _has_asc(platform) - inner = platform isa NoGPL ? platform.platform : platform - return !isa(inner, MacOS) && !isa(inner, Windows) -end - # Download a file, compute its hash, check for .asc signature, build metadata, # and write the updated versions.json. Returns true on success. function process_download!(meta, out_path, url, platform, version) @@ -252,21 +234,6 @@ function process_download!(meta, out_path, url, platform, version) ) end - # Test to see if there is an asc signature: - asc_signature = nothing - if _has_asc(platform) - asc_url = string(url, ".asc") - print(stdout, " Downloading $(basename(asc_url))") - try - asc_filepath = WebCacheUtilities.download_to_cache(basename(asc_url), asc_url) - asc_signature = String(read(asc_filepath)) - println(stdout, " ✓") - catch ex - isa(ex, InterruptException) && rethrow(ex) - println(stdout, " ✗") - end - end - # Build up metadata about this file if endswith(filename, ".dmg") kind = "archive" @@ -283,6 +250,22 @@ function process_download!(meta, out_path, url, platform, version) else error("Unsupported file extension in filename: $(filename)") end + + # Archives (.tar.gz, .zip) may have .asc signatures; installers (.dmg, .exe) don't + asc_signature = nothing + if extension in ("tar.gz", "zip") + asc_url = string(url, ".asc") + print(stdout, " Downloading $(basename(asc_url))") + try + asc_filepath = WebCacheUtilities.download_to_cache(basename(asc_url), asc_url) + asc_signature = String(read(asc_filepath)) + println(stdout, " ✓") + catch ex + isa(ex, InterruptException) && rethrow(ex) + println(stdout, " ✗") + end + end + file_dict = Dict( "triplet" => triplet(platform), "os" => meta_os(platform), @@ -294,7 +277,6 @@ function process_download!(meta, out_path, url, platform, version) "extension" => extension, "url" => url, ) - # Add in `.asc` signature content, if applicable if asc_signature !== nothing file_dict["asc"] = asc_signature end @@ -311,50 +293,40 @@ function process_download!(meta, out_path, url, platform, version) return true end -function main(out_path) - tags = get_tags() - tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) - +# Core loop: try downloading each (version, platform) URL and build metadata. +# `url_fn(version, platform)` returns the URL to try, or `nothing` to skip. +function _build_versions(out_path, tag_versions, platforms, url_fn) meta = Dict() number_urls_tried = 0 number_urls_success = 0 for version in tag_versions - for platform in julia_platforms - url = download_url(version, platform) + for platform in platforms + url = url_fn(version, platform) + url === nothing && continue number_urls_tried += 1 if process_download!(meta, out_path, url, platform, version) number_urls_success += 1 end end end - @info "Tried $(number_urls_tried) versions, successfully downloaded $(number_urls_success)" end +function main(out_path) + tags = get_tags() + tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) + _build_versions(out_path, tag_versions, julia_platforms, download_url) +end + function main_nogpl(out_path) tags = get_tags() tag_versions = filter(x -> x !== nothing, [vnum_maybe(basename(t["ref"])) for t in tags]) tag_commits = get_tag_commits() - - meta = Dict() - number_urls_tried = 0 - number_urls_success = 0 - for version in tag_versions + _build_versions(out_path, tag_versions, nogpl_platforms) do version, platform commit_hash = get(tag_commits, version, nothing) - if commit_hash === nothing - @info "No commit hash found for $(version), skipping nogpl builds" - continue - end - for platform in nogpl_platforms - url = download_url(version, platform, commit_hash) - number_urls_tried += 1 - if process_download!(meta, out_path, url, platform, version) - number_urls_success += 1 - end - end + commit_hash === nothing && return nothing + download_url(version, platform, commit_hash) end - - @info "Tried $(number_urls_tried) versions, successfully downloaded $(number_urls_success)" end end # module