From cecb86a69b2ad9602b6ba5cb7bba0cf2ffb4e89d Mon Sep 17 00:00:00 2001 From: Alexander Karan Date: Sat, 24 Jan 2026 19:16:16 +0800 Subject: [PATCH 1/4] Move build times to parallel --- .github/workflows/generate-stats.yml | 87 +++++++++++++++++++--------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 9c53ebf..3e15ccd 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -70,10 +70,57 @@ jobs: path: /tmp/framework-test/install-time.txt retention-days: 1 - # Build and other measurements + generate stats + # Build time measurements - parallel on fresh runners for fair comparison + measure-build: + needs: setup + if: needs.setup.outputs.build-matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(needs.setup.outputs.build-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + + - name: Measure ${{ matrix.framework.displayName }} build time + run: | + cp -r packages/${{ matrix.framework.package }} /tmp/framework-test + cd /tmp/framework-test + pnpm install + + START=$(date +%s%N) + pnpm build + END=$(date +%s%N) + COLD=$((($END - $START) / 1000000)) + + START=$(date +%s%N) + pnpm build + END=$(date +%s%N) + WARM=$((($END - $START) / 1000000)) + + echo "${{ matrix.framework.displayName }} - Cold: ${COLD}ms, Warm: ${WARM}ms" + echo "{\"cold\": $COLD, \"warm\": $WARM}" > build-time.json + + - name: Upload build time result + uses: actions/upload-artifact@v4 + with: + name: build-time-${{ matrix.framework.name }} + path: /tmp/framework-test/build-time.json + retention-days: 1 + + # Collect measurements + generate stats generate-stats: - needs: [setup, measure-install] - if: always() && needs.setup.result == 'success' + needs: [setup, measure-install, measure-build] + if: always() && needs.setup.result == 'success' && (needs.measure-install.result == 'success' || needs.measure-build.result == 'success') runs-on: ubuntu-latest permissions: contents: write @@ -101,29 +148,13 @@ jobs: path: artifacts pattern: install-time-* - - name: Measure build times - if: needs.setup.outputs.build-matrix != '[]' - run: | - BUILD_FRAMEWORKS=$(cat .github/frameworks.json | jq -c '[.[] | select(.measurements | contains(["build"]))]') - - echo "$BUILD_FRAMEWORKS" | jq -c '.[]' | while read -r framework; do - NAME=$(echo "$framework" | jq -r '.name') - DISPLAY_NAME=$(echo "$framework" | jq -r '.displayName') - BUILD_SCRIPT=$(echo "$framework" | jq -r '.buildScript') - - START=$(date +%s%N) - pnpm $BUILD_SCRIPT - END=$(date +%s%N) - COLD=$((($END - $START) / 1000000)) - - START=$(date +%s%N) - pnpm $BUILD_SCRIPT - END=$(date +%s%N) - WARM=$((($END - $START) / 1000000)) - - echo "$DISPLAY_NAME - Cold: ${COLD}ms, Warm: ${WARM}ms" - echo "{\"cold\": $COLD, \"warm\": $WARM}" > "/tmp/$NAME-build-time.json" - done + - name: Download build time artifacts + if: needs.setup.outputs.build-matrix != '[]' && needs.measure-build.result == 'success' + uses: actions/download-artifact@v4 + with: + path: artifacts + pattern: build-time-* + merge-multiple: false - name: Save CI stats run: | @@ -143,8 +174,8 @@ jobs: JSON="$JSON, \"installTimeMs\": $INSTALL_TIME" fi - if [[ "$MEASUREMENTS" == *"build"* ]] && [[ -f "/tmp/$NAME-build-time.json" ]]; then - BUILD_DATA=$(cat "/tmp/$NAME-build-time.json") + if [[ "$MEASUREMENTS" == *"build"* ]] && [[ -f "artifacts/build-time-$NAME/build-time.json" ]]; then + BUILD_DATA=$(cat "artifacts/build-time-$NAME/build-time.json") COLD=$(echo "$BUILD_DATA" | jq -r '.cold') WARM=$(echo "$BUILD_DATA" | jq -r '.warm') JSON="$JSON, \"coldBuildTimeMs\": $COLD, \"warmBuildTimeMs\": $WARM" From 949fa39927c7e0b50640619203279b2197895d4d Mon Sep 17 00:00:00 2001 From: Alexander Karan Date: Sun, 25 Jan 2026 07:47:36 +0800 Subject: [PATCH 2/4] Node Modules Size --- .github/workflows/generate-stats.yml | 40 +++++++++++++------ .../docs/src/components/DependencyStats.astro | 12 ++++++ packages/docs/src/content/config.ts | 2 + packages/stats-generator/src/types.ts | 2 + 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 9c53ebf..0ca5d41 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -52,22 +52,35 @@ jobs: with: node-version: '24' - - name: Measure ${{ matrix.framework.displayName }} install time + - name: Measure ${{ matrix.framework.displayName }} install time and sizes run: | cp -r packages/${{ matrix.framework.package }} /tmp/framework-test cd /tmp/framework-test + + # Measure install time (full install with dev dependencies) START=$(date +%s%N) pnpm install END=$(date +%s%N) - ELAPSED=$((($END - $START) / 1000000)) - echo "${{ matrix.framework.displayName }} install time: ${ELAPSED}ms" - echo "$ELAPSED" > install-time.txt + INSTALL_TIME=$((($END - $START) / 1000000)) + + # Measure node_modules size (in bytes) + SIZE=$(du -sb node_modules | cut -f1) + + # Clean and do production-only install + rm -rf node_modules + pnpm install --prod + + # Measure node_modules size production only (in bytes) + SIZE_PROD_ONLY=$(du -sb node_modules | cut -f1) + + echo "${{ matrix.framework.displayName }} - Install: ${INSTALL_TIME}ms, Size: ${SIZE} bytes, Prod only: ${SIZE_PROD_ONLY} bytes" + echo "{\"installTimeMs\": $INSTALL_TIME, \"nodeModulesSize\": $SIZE, \"nodeModulesSizeProdOnly\": $SIZE_PROD_ONLY}" > install-stats.json - - name: Upload install time result + - name: Upload install stats result uses: actions/upload-artifact@v4 with: - name: install-time-${{ matrix.framework.name }} - path: /tmp/framework-test/install-time.txt + name: install-stats-${{ matrix.framework.name }} + path: /tmp/framework-test/install-stats.json retention-days: 1 # Build and other measurements + generate stats @@ -94,12 +107,12 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Download install time artifacts + - name: Download install stats artifacts if: needs.setup.outputs.install-matrix != '[]' && needs.measure-install.result == 'success' uses: actions/download-artifact@v4 with: path: artifacts - pattern: install-time-* + pattern: install-stats-* - name: Measure build times if: needs.setup.outputs.build-matrix != '[]' @@ -138,9 +151,12 @@ jobs: JSON="{ \"timingMeasuredAt\": \"$TIMESTAMP\", \"runner\": \"ubuntu-latest\"" - if [[ "$MEASUREMENTS" == *"install"* ]] && [[ -f "artifacts/install-time-$NAME/install-time.txt" ]]; then - INSTALL_TIME=$(cat "artifacts/install-time-$NAME/install-time.txt") - JSON="$JSON, \"installTimeMs\": $INSTALL_TIME" + if [[ "$MEASUREMENTS" == *"install"* ]] && [[ -f "artifacts/install-stats-$NAME/install-stats.json" ]]; then + INSTALL_DATA=$(cat "artifacts/install-stats-$NAME/install-stats.json") + INSTALL_TIME=$(echo "$INSTALL_DATA" | jq -r '.installTimeMs') + SIZE=$(echo "$INSTALL_DATA" | jq -r '.nodeModulesSize') + SIZE_PROD_ONLY=$(echo "$INSTALL_DATA" | jq -r '.nodeModulesSizeProdOnly') + JSON="$JSON, \"installTimeMs\": $INSTALL_TIME, \"nodeModulesSize\": $SIZE, \"nodeModulesSizeProdOnly\": $SIZE_PROD_ONLY" fi if [[ "$MEASUREMENTS" == *"build"* ]] && [[ -f "/tmp/$NAME-build-time.json" ]]; then diff --git a/packages/docs/src/components/DependencyStats.astro b/packages/docs/src/components/DependencyStats.astro index 7299524..f7eebdc 100644 --- a/packages/docs/src/components/DependencyStats.astro +++ b/packages/docs/src/components/DependencyStats.astro @@ -3,6 +3,14 @@ import { getCollection } from 'astro:content' const statsEntries = await getCollection('stats') const stats = statsEntries.map((entry) => entry.data) + +const BYTES_PER_KB = 1024 +const BYTES_PER_MB = BYTES_PER_KB * BYTES_PER_KB + +function formatBytesToMB(bytes: number): string { + const mb = bytes / BYTES_PER_MB + return `${mb.toFixed(2)}MB` +} ---
@@ -35,6 +43,8 @@ const stats = statsEntries.map((entry) => entry.data) Framework Prod Deps Dev Deps + Size + Size (Prod Only) Graph @@ -45,6 +55,8 @@ const stats = statsEntries.map((entry) => entry.data) {framework.name} {framework.prodDependencies} {framework.devDependencies} + {formatBytesToMB(framework.nodeModulesSize)} + {formatBytesToMB(framework.nodeModulesSizeProdOnly)} Date: Sun, 25 Jan 2026 07:50:01 +0800 Subject: [PATCH 3/4] Pipeline clean up --- .github/workflows/generate-stats.yml | 42 ++++++++-------------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 62dfbdc..4d9c07b 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -154,54 +154,36 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Download install stats artifacts - if: needs.setup.outputs.install-matrix != '[]' && needs.measure-install.result == 'success' + - name: Download artifacts uses: actions/download-artifact@v4 with: path: artifacts - pattern: install-stats-* - - - name: Download build time artifacts - if: needs.setup.outputs.build-matrix != '[]' && needs.measure-build.result == 'success' - uses: actions/download-artifact@v4 - with: - path: artifacts - pattern: build-time-* - merge-multiple: false - name: Save CI stats run: | TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) - FRAMEWORKS=$(cat .github/frameworks.json) + BASE_STATS=$(jq -n --arg ts "$TIMESTAMP" '{timingMeasuredAt: $ts, runner: "ubuntu-latest"}') - echo "$FRAMEWORKS" | jq -c '.[]' | while read -r framework; do + jq -c '.[]' .github/frameworks.json | while read -r framework; do NAME=$(echo "$framework" | jq -r '.name') PACKAGE=$(echo "$framework" | jq -r '.package') DISPLAY_NAME=$(echo "$framework" | jq -r '.displayName') - MEASUREMENTS=$(echo "$framework" | jq -r '.measurements | join(",")') - JSON="{ \"timingMeasuredAt\": \"$TIMESTAMP\", \"runner\": \"ubuntu-latest\"" + STATS="$BASE_STATS" - if [[ "$MEASUREMENTS" == *"install"* ]] && [[ -f "artifacts/install-stats-$NAME/install-stats.json" ]]; then - INSTALL_DATA=$(cat "artifacts/install-stats-$NAME/install-stats.json") - INSTALL_TIME=$(echo "$INSTALL_DATA" | jq -r '.installTimeMs') - SIZE=$(echo "$INSTALL_DATA" | jq -r '.nodeModulesSize') - SIZE_PROD_ONLY=$(echo "$INSTALL_DATA" | jq -r '.nodeModulesSizeProdOnly') - JSON="$JSON, \"installTimeMs\": $INSTALL_TIME, \"nodeModulesSize\": $SIZE, \"nodeModulesSizeProdOnly\": $SIZE_PROD_ONLY" + INSTALL_FILE="artifacts/install-stats-$NAME/install-stats.json" + if [[ -f "$INSTALL_FILE" ]]; then + STATS=$(echo "$STATS" | jq --slurpfile install "$INSTALL_FILE" '. + $install[0]') fi - if [[ "$MEASUREMENTS" == *"build"* ]] && [[ -f "artifacts/build-time-$NAME/build-time.json" ]]; then - BUILD_DATA=$(cat "artifacts/build-time-$NAME/build-time.json") - COLD=$(echo "$BUILD_DATA" | jq -r '.cold') - WARM=$(echo "$BUILD_DATA" | jq -r '.warm') - JSON="$JSON, \"coldBuildTimeMs\": $COLD, \"warmBuildTimeMs\": $WARM" + BUILD_FILE="artifacts/build-time-$NAME/build-time.json" + if [[ -f "$BUILD_FILE" ]]; then + STATS=$(echo "$STATS" | jq --slurpfile build "$BUILD_FILE" '. + {coldBuildTimeMs: $build[0].cold, warmBuildTimeMs: $build[0].warm}') fi - JSON="$JSON }" - - echo "$DISPLAY_NAME stats: $JSON" + echo "$DISPLAY_NAME stats: $STATS" mkdir -p "packages/$PACKAGE" - echo "$JSON" | jq '.' > "packages/$PACKAGE/.ci-stats.json" + echo "$STATS" > "packages/$PACKAGE/.ci-stats.json" done - name: Generate stats From ce7c55a7ac208cd345a5ee48dd6b3b7c4d00510b Mon Sep 17 00:00:00 2001 From: Alexander Karan Date: Sun, 25 Jan 2026 08:18:35 +0800 Subject: [PATCH 4/4] Fixed type issues --- packages/docs/src/content/stats/starter-astro.json | 4 +++- packages/docs/src/content/stats/starter-next-js.json | 4 +++- packages/docs/src/content/stats/starter-nuxt.json | 4 +++- packages/docs/src/content/stats/starter-react-router.json | 4 +++- packages/docs/src/content/stats/starter-sveltekit.json | 4 +++- .../docs/src/content/stats/starter-tanstack-start-react.json | 4 +++- packages/starter-astro/package.json | 2 +- 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/docs/src/content/stats/starter-astro.json b/packages/docs/src/content/stats/starter-astro.json index 4927d71..b491efb 100644 --- a/packages/docs/src/content/stats/starter-astro.json +++ b/packages/docs/src/content/stats/starter-astro.json @@ -8,5 +8,7 @@ "runner": "ubuntu-latest", "installTimeMs": 3041, "coldBuildTimeMs": 2479, - "warmBuildTimeMs": 2422 + "warmBuildTimeMs": 2422, + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/docs/src/content/stats/starter-next-js.json b/packages/docs/src/content/stats/starter-next-js.json index bc0bbce..09e86c8 100644 --- a/packages/docs/src/content/stats/starter-next-js.json +++ b/packages/docs/src/content/stats/starter-next-js.json @@ -8,5 +8,7 @@ "coldBuildTimeMs": 7396, "warmBuildTimeMs": 7247, "timingMeasuredAt": "2026-01-24T09:51:21Z", - "runner": "ubuntu-latest" + "runner": "ubuntu-latest", + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/docs/src/content/stats/starter-nuxt.json b/packages/docs/src/content/stats/starter-nuxt.json index 23e322d..ee58372 100644 --- a/packages/docs/src/content/stats/starter-nuxt.json +++ b/packages/docs/src/content/stats/starter-nuxt.json @@ -8,5 +8,7 @@ "coldBuildTimeMs": 6582, "warmBuildTimeMs": 6370, "timingMeasuredAt": "2026-01-24T09:51:21Z", - "runner": "ubuntu-latest" + "runner": "ubuntu-latest", + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/docs/src/content/stats/starter-react-router.json b/packages/docs/src/content/stats/starter-react-router.json index f6b09a2..74e8fcf 100644 --- a/packages/docs/src/content/stats/starter-react-router.json +++ b/packages/docs/src/content/stats/starter-react-router.json @@ -8,5 +8,7 @@ "coldBuildTimeMs": 2983, "warmBuildTimeMs": 3007, "timingMeasuredAt": "2026-01-24T09:51:21Z", - "runner": "ubuntu-latest" + "runner": "ubuntu-latest", + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/docs/src/content/stats/starter-sveltekit.json b/packages/docs/src/content/stats/starter-sveltekit.json index fd00445..b691969 100644 --- a/packages/docs/src/content/stats/starter-sveltekit.json +++ b/packages/docs/src/content/stats/starter-sveltekit.json @@ -8,5 +8,7 @@ "coldBuildTimeMs": 4421, "warmBuildTimeMs": 4116, "timingMeasuredAt": "2026-01-24T09:51:21Z", - "runner": "ubuntu-latest" + "runner": "ubuntu-latest", + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/docs/src/content/stats/starter-tanstack-start-react.json b/packages/docs/src/content/stats/starter-tanstack-start-react.json index 116bb57..1bd0162 100644 --- a/packages/docs/src/content/stats/starter-tanstack-start-react.json +++ b/packages/docs/src/content/stats/starter-tanstack-start-react.json @@ -8,5 +8,7 @@ "coldBuildTimeMs": 9738, "warmBuildTimeMs": 9273, "timingMeasuredAt": "2026-01-24T09:51:21Z", - "runner": "ubuntu-latest" + "runner": "ubuntu-latest", + "nodeModulesSize": 0, + "nodeModulesSizeProdOnly": 0 } diff --git a/packages/starter-astro/package.json b/packages/starter-astro/package.json index b4e7599..2e977c6 100644 --- a/packages/starter-astro/package.json +++ b/packages/starter-astro/package.json @@ -9,7 +9,7 @@ "astro": "astro", "lint": "eslint .", "lint:fix": "eslint . --fix", - "type-check": "tsc --noEmit", + "type-check": "astro check", "format": "prettier --write .", "format:check": "prettier --check ." },