WIP: Start with the old versions.json, instead of starting from scratch

.github/dependabot.yml

@@ -1,10 +1,14 @@
 version: 2
 updates:
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "monthly"
     open-pull-requests-limit: 100
-    labels:
-      - "dependencies"
-      - "github-actions"
+
+  - package-ecosystem: "npm"
+    directory: "/test/node/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 100

.github/workflows/CI.yml

@@ -26,67 +26,61 @@ env:
 
 jobs:
   package-tests:
-    name: Package tests
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
         version:
-          - '1.10' # 1.10 = current LTS
-        os:
-          - ubuntu-latest
+          - '1.10' # currently the LTS
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
-
+        with:
+          persist-credentials: false
       - uses: julia-actions/setup-julia@4a12c5f801ca5ef0458bba44687563ef276522dd # v3.0.0
         with:
           version: ${{ matrix.version }}
-
       - uses: julia-actions/cache@9a93c5fb3e9c1c20b60fc80a478cae53e38618a4 # v3.0.2
-
       - name: Install dependencies
         run: julia --color=yes --project -e "using Pkg; Pkg.instantiate()"
-
       - uses: julia-actions/julia-runtest@d60b785c6f2bdf4ebfb18b2b6f7d93b7dfb0efe3 # v1.11.4
 
-  full-test:
-    name: Full test
+  full-build-and-test:
     # This job takes a long time (1+ hours).
     # So we intentionally skip this job on PR runs.
     # But we still run this job on Merge Queue jobs.
-    if: github.event_name != 'pull_request'
+    # if: github.event_name != 'pull_request' # TODO: Uncomment this line before merging the PR
     runs-on: ubuntu-latest
     timeout-minutes: 360
     strategy:
       fail-fast: false
-
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
-
+        with:
+          persist-credentials: false
       - uses: julia-actions/setup-julia@4a12c5f801ca5ef0458bba44687563ef276522dd # v3.0.0
         with:
-          version: '1.10' # 1.10 = current LTS
-
+          version: 'lts'
       - uses: julia-actions/cache@9a93c5fb3e9c1c20b60fc80a478cae53e38618a4 # v3.0.2
-
       - name: Install dependencies
         run: julia --color=yes --project -e "using Pkg; Pkg.instantiate()"
-
-      - run: rm -f versions.json
-
-      - name: Build versions.json
+      - name: Cleanup before we build
         run: |
-          using VersionsJSONUtil
-
-          VersionsJSONUtil.main("versions.json")
-        shell: julia --project {0}
-
+          make clean
+          make nuke
+          make purge_download_cache_dir
+      - name: Download the old versions.json, so we can re-use it and build on top of it
+        run: curl -LO https://julialang-s3.julialang.org/bin/versions.json
+      - name: Build versions.json
+        run: make build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Used to authenticate to GitHub API to get list of tags
+      - name: Run the post-build tests
+        run: make test-full
+      - name: Install NPM deps
+        run: make _npm_ci
       - name: Validate versions.json against schema
-        run: npx -p ajv-cli@3.3.0 ajv -s schema.json -d versions.json
-
-      - run: julia --project test/more_tests.jl versions.json
-
+        run: make check-schema
       - name: Upload versions.json as workflow artifact
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -95,31 +89,29 @@ jobs:
           if-no-files-found: error
 
   upload-to-s3:
-    needs: [package-tests, full-test]
+    needs:
+      - package-tests
+      - full-build-and-test
     if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
-
     steps:
       - name: Download versions.json from previous job
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: versions
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # 6.1.0
         with:
           aws-access-key-id: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }}
           aws-region: ${{ env.aws_region }}
-
       - name: Upload versions.json to S3
         run: aws s3 cp versions.json s3://${{ env.s3_bucket }}/bin/versions.json --acl public-read --no-progress
-
-      - name: Purge cache
+      - name: Purge Fastly cache (Fastly sits in front of S3)
         run: curl -X PURGE https://julialang-s3.julialang.org/bin/versions.json
 
-  test-current-s3:
+  test-current-s3-allowed-to-fail:
     # We intentionally do not make this job a "required status check" on PRs.
     # Because if the currently deployed `versions.json` on S3 is already broken,
     # then this job will fail, which would block us from merging a PR with fixes.
@@ -142,4 +134,4 @@ jobs:
       - name: Download the current versions.json from S3
         run: curl -LO https://julialang-s3.julialang.org/bin/versions.json
       - name: Run the post-build tests on the versions.json that we downloaded from S3
-        run: julia --project test/more_tests.jl versions.json
+        run: julia --project test/post_build.jl versions.json full

.gitignore

@@ -1 +1,10 @@
+# Directories:
+cache/
+node_modules/
+
+# Files:
+internal.json
 versions.json
+
+# macOS-specific:
+.DS_Store

Makefile

@@ -0,0 +1,60 @@
+# Usage:
+# make build
+# make test
+
+ifdef CI
+JULIA ?= julia
+else
+JULIA ?= julia +1.10
+endif
+
+# This is the default target:
+# It's a phony target because you might want to re-build versions.json even if none of the
+# files in this repo have changed.
+.PHONY: build
+build: _instantiate
+	JULIA_LOAD_PATH='@' $(JULIA) --startup-file=no --project -e 'import VersionsJSONUtil; VersionsJSONUtil.main(".")'
+
+# These are the post-build tests:
+.PHONY: test
+test: test-full
+
+.PHONY: test-full
+test-full: _instantiate
+	JULIA_LOAD_PATH='@:@stdlib' $(JULIA) --startup-file=no --project test/post_build.jl versions.json full
+
+.PHONY: test-partial
+test-partial: _instantiate
+	JULIA_LOAD_PATH='@:@stdlib' $(JULIA) --startup-file=no --project test/post_build.jl versions.json partial
+
+.PHONY: check-schema
+check-schema:
+	(cd test/node && npx ajv -s ../../schema.json -d ../../versions.json)
+
+# ------------------------------------------------------------------------------------------
+
+.PHONY: _instantiate
+_instantiate:
+	JULIA_LOAD_PATH='@:@stdlib' $(JULIA) --startup-file=no --project -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+.PHONY: _npm_ci
+_npm_ci:
+	(cd test/node && npm ci)
+
+
+.PHONY: clean
+clean:
+	$(RM) -v internal.json
+
+.PHONY: nuke
+nuke:
+	$(RM) -v internal.json
+	$(RM) -v versions.json
+
+.PHONY: _purge_node_modules
+_purge_node_modules:
+	$(RM) -r test/node/node_modules/
+
+.PHONY: purge_download_cache_dir
+purge_download_cache_dir:
+	$(RM) -r cache/