diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0e07409471b..6aacbfb4001 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -312,6 +312,9 @@ jobs: extension_tests_win: name: Run VS Code extension tests (Windows) runs-on: windows-latest + env: + NPM_REGISTRY: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ + COREPACK_ENABLE_DOWNLOAD_PROMPT: 0 defaults: run: working-directory: ./extension @@ -322,23 +325,126 @@ jobs: uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: '20.x' + - name: Install Corepack + run: | + # Scope Corepack's cache to this job so prepareCorepackYarn.mjs cannot + # collide with any other build sharing the runner's user profile. + # Cannot be set at job-level env: the `runner` context is unavailable + # in job env evaluation, so we forward it through $GITHUB_ENV instead. + $CorepackHome = Join-Path $env:RUNNER_TEMP 'corepack' + $env:COREPACK_HOME = $CorepackHome + "COREPACK_HOME=$CorepackHome" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + $CorepackVersion = (Get-Content -Raw -Path 'scripts/corepack-version.txt').Trim() + # The hosted Windows image already has a yarn shim in npm's global + # prefix. The npm Corepack package owns that shim too, so force only + # in CI where the tool install is isolated to this ephemeral job. + npm install --global --force --registry "$env:NPM_REGISTRY" "corepack@$CorepackVersion" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $npmGlobalBin = (npm prefix --global).Trim() + $env:PATH = "$npmGlobalBin$([IO.Path]::PathSeparator)$env:PATH" + $npmGlobalBin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + $installed = (corepack --version).Trim() + if ($installed -ne $CorepackVersion) { + Write-Error "corepack version mismatch: expected $CorepackVersion, got '$installed'. The bundled Corepack on PATH may be taking precedence over the npm-global install." + exit 1 + } + + corepack enable + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Seed Corepack's cache from the internal dnceng npm feed. Corepack's + # built-in `corepack prepare --activate` would download Yarn 1.x from + # registry.yarnpkg.com (hardcoded in Corepack 0.34's config.json and + # not redirectable via COREPACK_NPM_REGISTRY), bypassing the dnceng + # mirror this workflow is supposed to validate. The shared + # prepareCorepackYarn.mjs script does the equivalent via `npm pack` + # against $NPM_REGISTRY, then drops the same on-disk layout Corepack + # would have written. + node ./scripts/prepareCorepackYarn.mjs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + corepack yarn --version - name: Validate lockfile registries - run: node -e "const fs = require('fs'); const lock = fs.readFileSync('yarn.lock', 'utf8'); if (/registry\\.(?:npmjs\\.org|yarnpkg\\.com)/.test(lock)) { throw new Error('extension/yarn.lock contains public npm registry URLs. Regenerate it using the internal dotnet-public-npm feed before restoring.'); }" + # Allowlist scoped to lines starting with "resolved" (the only lines in + # yarn.lock that carry a tarball URL — `npm:` aliases don't). + # CONTRIBUTING.MD asks contributors to regenerate yarn.lock through the + # internal dotnet-public-npm feed; an allowlist catches drift to any + # other public mirror (npmmirror.com, jsr.io, github.com tarballs, ...) + # rather than only npmjs.org and yarnpkg.com. + run: | + node -e "const fs = require('fs'); const allow = 'pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm'; const bad = fs.readFileSync('yarn.lock', 'utf8').split(/\r?\n/).filter(l => /^\s*resolved\s+\x22/.test(l)).filter(l => !l.includes(allow)); if (bad.length) { throw new Error('extension/yarn.lock contains resolved entries outside the internal dotnet-public-npm feed. Regenerate it through the internal feed before restoring. First offender -> ' + bad[0]); }" - name: Install dependencies - run: yarn install --frozen-lockfile --non-interactive + run: corepack yarn install --frozen-lockfile --non-interactive - name: Run tests - run: yarn test + run: corepack yarn test - name: Override extension version for PR builds if: ${{ inputs.extensionVersionOverride != '' }} - run: yarn version --new-version "${{ inputs.extensionVersionOverride }}" --no-git-tag-version + run: corepack yarn version --new-version "${{ inputs.extensionVersionOverride }}" --no-git-tag-version - name: Package VSIX - run: yarn run vsce package --pre-release -o out/aspire-extension.vsix + run: corepack yarn run vsce package --pre-release -o out/aspire-extension.vsix - name: Upload VSIX uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: aspire-extension path: extension/out/aspire-extension.vsix + extension_bootstrap_linux: + name: Validate VS Code extension bootstrap (Linux) + # Without this, the non-Windows code paths in + # extension/scripts/prepareCorepackYarn.mjs (npm invocation that does not go + # through node.exe, POSIX tar) are only exercised on contributor machines + # and never on a fresh CI image. extension_tests_win covers the Windows + # paths; this job covers Linux/macOS. macOS is omitted because the only + # platform-specific branch beyond Linux is the cache path, which is + # explicitly overridden via COREPACK_HOME in the build entrypoints. + runs-on: ubuntu-latest + env: + NPM_REGISTRY: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ + COREPACK_ENABLE_DOWNLOAD_PROMPT: 0 + defaults: + run: + working-directory: ./extension + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup Node.js environment + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: '20.x' + - name: Install Corepack + run: | + set -euo pipefail + # Scope Corepack's cache to this job. Set via $GITHUB_ENV because the + # `runner` context is not available in job-level env evaluation. + export COREPACK_HOME="$RUNNER_TEMP/corepack" + echo "COREPACK_HOME=$COREPACK_HOME" >> "$GITHUB_ENV" + + CorepackVersion="$(tr -d '[:space:]' < scripts/corepack-version.txt)" + npm install --global --force --registry "$NPM_REGISTRY" "corepack@${CorepackVersion}" + + installed="$(corepack --version 2>/dev/null || true)" + if [ "$installed" != "$CorepackVersion" ]; then + echo "corepack version mismatch: expected $CorepackVersion, got '$installed'. The bundled Corepack on PATH may be taking precedence over the npm-global install." + exit 1 + fi + + corepack enable + - name: Seed Corepack Yarn cache via prepareCorepackYarn.mjs + # Exercises the script's non-Windows branches against a clean cache + # directory. Failing here means a contributor on Linux/macOS following + # CONTRIBUTING.MD would also be broken. + run: node ./scripts/prepareCorepackYarn.mjs + - name: Validate lockfile registries + # Mirror of the same allowlist guard in extension_tests_win so this + # cross-platform bootstrap also catches drift to a public-registry URL. + run: | + node -e "const fs = require('fs'); const allow = 'pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm'; const bad = fs.readFileSync('yarn.lock', 'utf8').split(/\r?\n/).filter(l => /^\s*resolved\s+\x22/.test(l)).filter(l => !l.includes(allow)); if (bad.length) { throw new Error('extension/yarn.lock contains resolved entries outside the internal dotnet-public-npm feed. Regenerate it through the internal feed before restoring. First offender -> ' + bad[0]); }" + - name: Validate seeded cache via yarn install + run: corepack yarn install --frozen-lockfile --non-interactive + typescript_sdk_tests: name: TypeScript SDK Unit Tests uses: ./.github/workflows/typescript-sdk-tests.yml @@ -364,6 +470,7 @@ jobs: prepare_homebrew_installer_artifacts, build_cli_e2e_image, extension_tests_win, + extension_bootstrap_linux, cli_starter_validation_windows, typescript_sdk_tests, typescript_api_compat, diff --git a/.gitignore b/.gitignore index 60371d10272..edfd6e9b891 100644 --- a/.gitignore +++ b/.gitignore @@ -197,6 +197,7 @@ extension/dist/ extension/.localization/ extension/out/ extension/node_modules/ +extension/.corepack-cache/ **/.vscode-test/ extension/.version diff --git a/eng/pipelines/azure-pipelines-codeql.yml b/eng/pipelines/azure-pipelines-codeql.yml index 5863fffbd4d..273802c5170 100644 --- a/eng/pipelines/azure-pipelines-codeql.yml +++ b/eng/pipelines/azure-pipelines-codeql.yml @@ -56,14 +56,18 @@ jobs: inputs: version: '20.x' + - task: npmAuthenticate@0 + displayName: NPM authenticate + inputs: + workingFile: $(Build.SourcesDirectory)\.npmrc + - task: PowerShell@2 - displayName: Install yarn + displayName: Set .npmrc environment inputs: targetType: 'inline' - script: | - npm install -g yarn@1.22.22 - yarn --version - workingDirectory: '$(Build.SourcesDirectory)' + script: Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$(Build.SourcesDirectory)\.npmrc" + + - template: /eng/pipelines/templates/install-corepack.yml - task: PowerShell@2 displayName: Install vsce diff --git a/eng/pipelines/azure-pipelines-unofficial.yml b/eng/pipelines/azure-pipelines-unofficial.yml index 0e93eb5bfd3..06948dd273c 100644 --- a/eng/pipelines/azure-pipelines-unofficial.yml +++ b/eng/pipelines/azure-pipelines-unofficial.yml @@ -209,14 +209,9 @@ extends: targetType: 'inline' script: Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$(Build.SourcesDirectory)\.npmrc" - - task: PowerShell@2 - displayName: 🟣Install yarn - inputs: - targetType: 'inline' - script: | - npm install -g yarn@1.22.22 - yarn --version - workingDirectory: '$(Build.SourcesDirectory)' + - template: /eng/pipelines/templates/install-corepack.yml + parameters: + displayPrefix: '🟣' - task: PowerShell@2 displayName: 🟣Install vsce diff --git a/eng/pipelines/azure-pipelines.yml b/eng/pipelines/azure-pipelines.yml index 9184a94e07a..bb0ba783ba1 100644 --- a/eng/pipelines/azure-pipelines.yml +++ b/eng/pipelines/azure-pipelines.yml @@ -298,14 +298,9 @@ extends: targetType: 'inline' script: Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$(Build.SourcesDirectory)\.npmrc" - - task: PowerShell@2 - displayName: 🟣Install yarn - inputs: - targetType: 'inline' - script: | - npm install -g yarn@1.22.22 - yarn --version - workingDirectory: '$(Build.SourcesDirectory)' + - template: /eng/pipelines/templates/install-corepack.yml + parameters: + displayPrefix: '🟣' - task: PowerShell@2 displayName: 🟣Install vsce diff --git a/eng/pipelines/common-variables.yml b/eng/pipelines/common-variables.yml index d5852f31656..30d5b0cbcc0 100644 --- a/eng/pipelines/common-variables.yml +++ b/eng/pipelines/common-variables.yml @@ -10,6 +10,15 @@ variables: - name: _InternalBuildArgs value: '' + # npm global installs and the Corepack Yarn cache seeder don't use the repo + # .npmrc, so pass this registry explicitly. + - name: NPM_REGISTRY + value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ + # Disable the interactive "Do you want to download yarn@x.y.z?" prompt so + # pipeline steps don't hang waiting for stdin. + - name: COREPACK_ENABLE_DOWNLOAD_PROMPT + value: '0' + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - name: _RunAsPublic value: False diff --git a/eng/pipelines/templates/install-corepack.yml b/eng/pipelines/templates/install-corepack.yml new file mode 100644 index 00000000000..efbaa0b7359 --- /dev/null +++ b/eng/pipelines/templates/install-corepack.yml @@ -0,0 +1,55 @@ +parameters: + # Prefix prepended to step displayNames (e.g. the 🟣 emoji used by the + # internal-1ES jobs in azure-pipelines.yml / azure-pipelines-unofficial.yml). + # Public/CodeQL pipelines omit it. Kept as a parameter so the displayName + # convention of each calling pipeline is preserved. + - name: displayPrefix + type: string + default: '' + +steps: + - task: PowerShell@2 + displayName: ${{ parameters.displayPrefix }}Install Corepack + inputs: + targetType: 'inline' + script: | + # Pinned Corepack shim version, sourced from + # extension/scripts/corepack-version.txt so this script, + # extension/build.sh, extension/build.ps1, the GitHub Actions + # workflow, and every AzDO pipeline that references this template + # stay in sync. The Yarn release itself is pinned in + # extension/package.json via the `packageManager` field. + $CorepackVersion = (Get-Content -Raw -Path '$(Build.SourcesDirectory)/extension/scripts/corepack-version.txt').Trim() + $corepackHome = Join-Path '$(Agent.TempDirectory)' 'corepack' + $env:COREPACK_HOME = $corepackHome + Write-Host "##vso[task.setvariable variable=COREPACK_HOME]$corepackHome" + + # Hosted Windows images can already have yarn in npm's global prefix. + # The npm Corepack package owns that shim too, so force only in CI + # where the tool install is job-scoped. + npm install -g --force --registry "$env:NPM_REGISTRY" "corepack@$CorepackVersion" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $npmGlobalBin = (npm prefix --global).Trim() + $env:PATH = "$npmGlobalBin$([IO.Path]::PathSeparator)$env:PATH" + Write-Host "##vso[task.prependpath]$npmGlobalBin" + + # Verify the version actually on PATH matches our pin. On Windows the + # Node.js installer registers a `corepack.cmd` under + # %ProgramFiles%\nodejs which may shadow the npm-global shim under + # %APPDATA%\npm, so a successful `npm install -g corepack@` + # does NOT guarantee that subsequent `corepack` calls resolve to it. + $installed = (corepack --version).Trim() + if ($installed -ne $CorepackVersion) { + Write-Error "corepack version mismatch: expected $CorepackVersion, got '$installed'. The bundled Corepack on PATH may be taking precedence over the npm-global install." + exit 1 + } + + corepack enable + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + node ./scripts/prepareCorepackYarn.mjs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + corepack yarn --version + workingDirectory: '$(Build.SourcesDirectory)\extension' diff --git a/eng/pipelines/templates/public-pipeline-template.yml b/eng/pipelines/templates/public-pipeline-template.yml index e560f93ed19..e200857acde 100644 --- a/eng/pipelines/templates/public-pipeline-template.yml +++ b/eng/pipelines/templates/public-pipeline-template.yml @@ -32,6 +32,15 @@ variables: - name: _InternalBuildArgs value: '' + # npm global installs and the Corepack Yarn cache seeder don't use the repo + # .npmrc, so pass this registry explicitly. + - name: NPM_REGISTRY + value: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ + # Disable the interactive "Do you want to download yarn@x.y.z?" prompt so + # pipeline steps don't hang waiting for stdin. + - name: COREPACK_ENABLE_DOWNLOAD_PROMPT + value: '0' + # Set test variants based on parameter or build reason - ${{ if ne(parameters.testVariants, '') }}: - name: testVariants diff --git a/extension/CONTRIBUTING.MD b/extension/CONTRIBUTING.MD index 41e9a739728..905192acca0 100644 --- a/extension/CONTRIBUTING.MD +++ b/extension/CONTRIBUTING.MD @@ -2,8 +2,11 @@ ## Install Prerequisites -- Node.js (LTS version) -- Yarn accessible in the PATH +- Node.js (LTS version) — `npm` must be on the PATH (it ships with Node.js). The + build scripts (`build.sh` / `build.ps1`) install a pinned [Corepack](https://github.com/nodejs/corepack) + via `npm install -g corepack@` from the configured registry and seed + Corepack's cache with the Yarn release pinned by the `packageManager` field in + `extension/package.json`. You do **not** need to install Yarn yourself. - Visual Studio Code (latest) or Visual Studio Code Insiders - [Aspire CLI](https://aspire.dev/get-started/install-cli/) must be installed and available in the PATH @@ -26,12 +29,39 @@ You can use the `Aspire: Extension settings` command to open VS Code settings di ## Updating dependency overrides -The extension is built with **yarn**, so `package.json` uses `resolutions` for transitive dependency pins and `yarn.lock` is the authoritative lockfile. +The extension is built with **yarn**, pinned to the version recorded in `packageManager` of `package.json`. `package.json` uses `resolutions` for transitive dependency pins and `yarn.lock` is the authoritative lockfile. When pinning a transitive dependency (e.g. to address a security advisory), add the pin to `resolutions` and regenerate `yarn.lock` in the same change: ```bash -yarn install +corepack yarn install ``` -The build rejects public npm registry URLs in `yarn.lock`; ensure regenerated entries resolve through the internal `dotnet-public-npm` feed. \ No newline at end of file +The build rejects public npm registry URLs in `yarn.lock`; ensure regenerated entries resolve through the internal `dotnet-public-npm` feed. + +## Updating the Yarn version + +Edit the `"packageManager": "yarn@x.y.z"` field in `extension/package.json`. The next `build.sh` / `build.ps1` run seeds Corepack's cache with that version before calling `corepack yarn …`. No further changes are required in `build.sh`, `build.ps1`, or `extension/Extension.proj`. + +> **Heads up about the internal npm mirror.** The repo's `.npmrc` and the build scripts' `NPM_REGISTRY` default route npm package downloads through the dnceng `dotnet-public-npm` Azure Artifacts feed, which is a pull-through cache of npmjs.org. The first time anyone asks the feed for a version that has never been requested, the feed has to fetch it from npmjs.org — and anonymous pull-through fetches fail with HTTP 401. Subsequent anonymous reads work fine. If you bump the pinned Corepack or Yarn version, pre-seed the feed with credentials using `npm install --global --registry https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ corepack@` or `npm pack --registry https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/ yarn@`. Do not point `COREPACK_NPM_REGISTRY` at this Azure Artifacts feed for Yarn: Corepack requests the `//` npm metadata route, and Azure Artifacts returns 404 for that route even when the package and tarball exist. + +## Troubleshooting + +### `EACCES` from `npm install --global` on Linux/macOS + +The build scripts run `npm install --global corepack@` to pin the Corepack version. On systems where Node.js is installed from a package manager (apt, yum, the official `.pkg`), the npm global prefix is typically `/usr/lib/node_modules` or `/usr/local/lib/node_modules`, which is root-owned. The install will fail with `EACCES`. The cleanest fix is to use a Node version manager that puts the npm prefix in your home directory: + +- [`nvm`](https://github.com/nvm-sh/nvm) — installs Node and configures the npm prefix automatically. +- [`fnm`](https://github.com/Schniz/fnm), [`asdf`](https://asdf-vm.com/), [`volta`](https://volta.sh/) — same idea, different tradeoffs. + +Alternatively, point npm at a user-writable prefix without changing your Node install: + +```bash +mkdir -p ~/.npm-global +npm config set prefix ~/.npm-global +export PATH="$HOME/.npm-global/bin:$PATH" # add to ~/.bashrc or ~/.zshrc +``` + +### `corepack version mismatch` from `build.sh` / `build.ps1` + +This means the `corepack` resolved from `PATH` is not the one we just installed via `npm install -g`. Most often the system Node install (`/usr/bin/corepack`, `%ProgramFiles%\nodejs\corepack.cmd`) is sitting in front of the npm global bin directory. On Windows, ensure `%APPDATA%\npm` comes before `%ProgramFiles%\nodejs` on `PATH`. On Linux/macOS, follow the `EACCES` remediation above and the npm prefix will be on `PATH` ahead of the system Node directory. diff --git a/extension/Extension.proj b/extension/Extension.proj index 5436af60c54..c6e350aad2b 100644 --- a/extension/Extension.proj +++ b/extension/Extension.proj @@ -2,6 +2,31 @@ $(DefaultTargetFramework) $(MSBuildThisFileDirectory) + + <_CorepackHome Condition="'$(COREPACK_HOME)' != ''">$(COREPACK_HOME) + <_CorepackHome Condition="'$(_CorepackHome)' == ''">$([MSBuild]::NormalizePath($(ExtensionSrcDir), '.corepack-cache')) @@ -31,13 +56,14 @@ - - - + + @@ -70,6 +96,14 @@ <_YarnLockPath>$([MSBuild]::NormalizePath($(ExtensionSrcDir), 'yarn.lock')) + + <_AllowedNpmRegistryHost>pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm @@ -79,21 +113,28 @@ - <_PublicNpmRegistryLine Include="@(_YarnLockLine)" Condition="$([System.String]::Copy('%(_YarnLockLine.Identity)').Contains('registry.npmjs.org')) Or $([System.String]::Copy('%(_YarnLockLine.Identity)').Contains('registry.yarnpkg.com'))" /> + <_YarnLockResolvedLine Include="@(_YarnLockLine)" Condition="$([System.String]::Copy('%(_YarnLockLine.Identity)').TrimStart().StartsWith('resolved "'))" /> + <_DisallowedRegistryLine Include="@(_YarnLockResolvedLine)" Condition="!$([System.String]::Copy('%(_YarnLockResolvedLine.Identity)').Contains('$(_AllowedNpmRegistryHost)'))" /> + Text="extension/yarn.lock contains resolved entries outside the internal dotnet-public-npm feed. Regenerate it through the internal feed before building. Offending lines: @(_DisallowedRegistryLine, '; ')" + Condition="'@(_DisallowedRegistryLine)' != ''" /> - + + - + diff --git a/extension/build.ps1 b/extension/build.ps1 index ccd4be4305b..94436d77491 100644 --- a/extension/build.ps1 +++ b/extension/build.ps1 @@ -2,6 +2,54 @@ $ErrorActionPreference = "Stop" +# Ensure we run from the extension directory so corepack/yarn pick up +# extension/.npmrc and extension/package.json (which holds the packageManager pin). +Set-Location $PSScriptRoot + +# Pinned Corepack shim version. Node.js >= 16.10 bundles a Corepack, but the +# bundled version drifts with each Node release and Corepack is on track to be +# unbundled from Node entirely (see https://github.com/nodejs/node/issues/54647). +# Installing a pinned Corepack from npm makes the build reproducible regardless +# of which Node version a developer or CI runner happens to have. +# +# The version is stored in scripts/corepack-version.txt so that this script, the +# Bash build script, the GitHub Actions workflow, and the AzDO pipelines all +# read from a single source of truth. +$CorepackVersion = (Get-Content -Raw -Path (Join-Path $PSScriptRoot 'scripts/corepack-version.txt')).Trim() +if ([string]::IsNullOrWhiteSpace($CorepackVersion)) { + Write-Error "scripts/corepack-version.txt is empty or unreadable." + exit 1 +} + +# Yarn version is pinned in extension/package.json via the "packageManager" +# field, which scripts/prepareCorepackYarn.mjs uses to seed Corepack's cache. + +# Point npm at the dnceng internal npm mirror when installing the pinned Corepack +# shim and seeding Corepack's Yarn cache. npm global installs do not use the +# project .npmrc, so pass the registry explicitly. Override locally with +# `$env:NPM_REGISTRY = ''; ./build.ps1`. +if (-not $env:NPM_REGISTRY) { + $env:NPM_REGISTRY = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/" +} +if (-not $env:COREPACK_ENABLE_DOWNLOAD_PROMPT) { + $env:COREPACK_ENABLE_DOWNLOAD_PROMPT = "0" +} + +# Pin Corepack's cache directory to a build-scoped location. Without this, every +# build shares the user's default cache (%LOCALAPPDATA%\node\corepack on +# Windows, ~/.cache/node/corepack on Linux, ~/Library/Caches/node/corepack on +# macOS). prepareCorepackYarn.mjs rewrites that cache in place +# (rmSync(installDirectory) followed by renameSync(staging, installDirectory)), +# so concurrent builds racing on the same Corepack home can corrupt each other's +# cache. The CI pipelines already scope this per-job (e.g. AzDO uses +# Agent.TempDirectory); do the same here so multi-worktree setups stay +# isolated. Concurrent builds in the *same* worktree still race on this shared +# directory — prepareCorepackYarn.mjs handles the EEXIST/ENOTEMPTY rename +# collision but is not a substitute for a lock. The directory is gitignored. +if (-not $env:COREPACK_HOME) { + $env:COREPACK_HOME = Join-Path $PSScriptRoot '.corepack-cache' +} + Write-Host "Checking prerequisites..." # Check for Node.js @@ -10,9 +58,10 @@ if (-not (Get-Command node -ErrorAction SilentlyContinue)) { exit 1 } -# Check for Corepack so the build uses the Yarn Classic version that matches extension/yarn.lock. -if (-not (Get-Command corepack -ErrorAction SilentlyContinue)) { - Write-Error "Error: Corepack is not installed. Please install a Node.js version that includes Corepack." +# npm is required to install our pinned Corepack. It ships with every official +# Node.js distribution, so this should only fail on broken installs. +if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + Write-Error "Error: npm is not available. Reinstall Node.js so npm is on PATH." exit 1 } @@ -35,12 +84,65 @@ if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { Write-Host "All prerequisites satisfied." -# Ensure we run from the extension directory -Set-Location $PSScriptRoot +Write-Host "" +Write-Host "Installing pinned Corepack $CorepackVersion..." +# Reinstall every time so we overwrite any older Corepack shim that Node.js +# may have placed on PATH ahead of npm's global prefix. npm global installs do +# not use the project .npmrc, so pass the registry explicitly. +# +# --force is required because Corepack's npm package declares `yarn`, `yarnpkg`, +# `pnpm`, `pnpx`, and `corepack` as bin entries. Without --force, npm refuses to +# overwrite bins owned by a pre-existing global yarn or pnpm (a very common +# setup, and the state this repo itself was in before this build script existed) +# and aborts with EEXIST. The CI pipelines already pass --force for the same +# reason. +npm install --global --force --registry "$env:NPM_REGISTRY" "corepack@$CorepackVersion" + +if ($LASTEXITCODE -ne 0) { + Write-Error "npm install -g corepack@$CorepackVersion failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +# Verify the version actually on PATH matches our pin. On Windows the Node.js +# installer registers a `corepack.cmd` under %ProgramFiles%\nodejs which may +# shadow the npm-global shim under %APPDATA%\npm, so a successful +# `npm install -g corepack@` does NOT guarantee that subsequent +# `corepack` calls resolve to it. Fail loudly here so we don't silently run +# with the wrong tool. +$installedCorepack = $null +try { $installedCorepack = (corepack --version).Trim() } catch { } +if ($installedCorepack -ne $CorepackVersion) { + Write-Error @" +corepack version mismatch: expected $CorepackVersion, got '$installedCorepack'. +The bundled Corepack on PATH may be taking precedence over the npm-global install. +Ensure your npm global bin directory (typically %APPDATA%\npm on Windows) comes +before %ProgramFiles%\nodejs on PATH, or run `corepack disable` to remove the +bundled shim before re-running this script. +"@ + exit 1 +} + +Write-Host "" +Write-Host "Enabling Corepack package manager shims..." +corepack enable + +if ($LASTEXITCODE -ne 0) { + Write-Error "corepack enable failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "Preparing Yarn from packageManager pin in package.json..." +node ./scripts/prepareCorepackYarn.mjs + +if ($LASTEXITCODE -ne 0) { + Write-Error "Preparing Yarn for Corepack failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "" Write-Host "Running yarn install..." -corepack yarn@1.22.22 install --frozen-lockfile --non-interactive +corepack yarn install --frozen-lockfile --non-interactive if ($LASTEXITCODE -ne 0) { Write-Error "yarn install failed with exit code $LASTEXITCODE" @@ -49,7 +151,7 @@ if ($LASTEXITCODE -ne 0) { Write-Host "" Write-Host "Running yarn compile..." -corepack yarn@1.22.22 compile +corepack yarn compile if ($LASTEXITCODE -ne 0) { Write-Error "yarn compile failed with exit code $LASTEXITCODE" diff --git a/extension/build.sh b/extension/build.sh index 110967f4215..1fbb702d5ce 100755 --- a/extension/build.sh +++ b/extension/build.sh @@ -1,6 +1,55 @@ #!/bin/bash set -e +# Ensure we run from the extension directory so corepack/yarn pick up +# extension/.npmrc and extension/package.json (which holds the packageManager pin). +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Pinned Corepack shim version. Node.js >= 16.10 bundles a Corepack, but the +# bundled version drifts with each Node release and Corepack is on track to be +# unbundled from Node entirely (see https://github.com/nodejs/node/issues/54647). +# Installing a pinned Corepack from npm makes the build reproducible regardless +# of which Node version a developer or CI runner happens to have. +# +# The version is stored in scripts/corepack-version.txt so that this script, the +# PowerShell build script, the GitHub Actions workflow, and the AzDO pipelines +# all read from a single source of truth. +COREPACK_VERSION="$(tr -d '[:space:]' < "$SCRIPT_DIR/scripts/corepack-version.txt")" +if [ -z "$COREPACK_VERSION" ]; then + echo "Error: scripts/corepack-version.txt is empty or unreadable." + exit 1 +fi + +# Yarn version is pinned in extension/package.json via the "packageManager" +# field, which scripts/prepareCorepackYarn.mjs uses to seed Corepack's cache. + +# Point npm at the dnceng internal npm mirror when installing the pinned Corepack +# shim and seeding Corepack's Yarn cache. npm global installs do not use the +# project .npmrc, so pass the registry explicitly. Override locally with +# `NPM_REGISTRY= ./build.sh`. +# Export NPM_REGISTRY so the child `node ./scripts/prepareCorepackYarn.mjs` +# process inherits it; without `export`, the script silently falls back to its +# own DefaultNpmRegistry constant and any user override of NPM_REGISTRY would +# be ignored when seeding Corepack's Yarn cache. +: "${NPM_REGISTRY:=https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/}" +export NPM_REGISTRY +: "${COREPACK_ENABLE_DOWNLOAD_PROMPT:=0}" +export COREPACK_ENABLE_DOWNLOAD_PROMPT + +# Pin Corepack's cache directory to a build-scoped location. Without this, every +# build shares the user's default cache (~/.cache/node/corepack on Linux, +# ~/Library/Caches/node/corepack on macOS, %LOCALAPPDATA%\node\corepack on +# Windows). prepareCorepackYarn.mjs rewrites that cache in place +# (rmSync(installDirectory) followed by renameSync(staging, installDirectory)), +# so concurrent builds racing on the same Corepack home can corrupt each other's +# cache. The CI pipelines already scope this per-job (e.g. AzDO uses +# Agent.TempDirectory); do the same here so multi-worktree setups stay +# isolated. Concurrent builds in the *same* worktree still race on this shared +# directory — prepareCorepackYarn.mjs handles the EEXIST/ENOTEMPTY rename +# collision but is not a substitute for a lock. The directory is gitignored. +: "${COREPACK_HOME:=$SCRIPT_DIR/.corepack-cache}" +export COREPACK_HOME + echo "Checking prerequisites..." # Check for Node.js @@ -9,9 +58,10 @@ if ! command -v node &> /dev/null; then exit 1 fi -# Check for Corepack so the build uses the Yarn Classic version that matches extension/yarn.lock. -if ! command -v corepack &> /dev/null; then - echo "Error: Corepack is not installed. Please install a Node.js version that includes Corepack." +# npm is required to install our pinned Corepack. It ships with every official +# Node.js distribution, so this should only fail on broken installs. +if ! command -v npm &> /dev/null; then + echo "Error: npm is not available. Reinstall Node.js so npm is on PATH." exit 1 fi @@ -31,17 +81,51 @@ fi echo "All prerequisites satisfied." -# Ensure we run from the extension directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" +echo "" +echo "Installing pinned Corepack ${COREPACK_VERSION}..." +# Reinstall every time so we overwrite any older Corepack shim that Node.js +# may have placed on PATH ahead of npm's global prefix. npm global installs do +# not use the project .npmrc, so pass the registry explicitly. +# +# --force is required because Corepack's npm package declares `yarn`, `yarnpkg`, +# `pnpm`, `pnpx`, and `corepack` as bin entries. Without --force, npm refuses to +# overwrite bins owned by a pre-existing global yarn or pnpm (a very common +# setup, and the state this repo itself was in before this build script existed) +# and aborts with EEXIST. The CI pipelines already pass --force for the same +# reason. +npm install --global --force --registry "$NPM_REGISTRY" "corepack@${COREPACK_VERSION}" + +# Verify the version actually on PATH matches our pin. If a system-bundled +# Corepack shim shadows the npm-global install (common on Windows; possible on +# macOS/Linux when /usr/local/bin precedes the npm prefix), `npm install -g` +# can "succeed" while subsequent `corepack` calls still resolve to the bundled +# version. Fail loudly here so we don't silently run with the wrong tool. +installed_corepack=$(corepack --version 2>/dev/null || echo "") +if [ "$installed_corepack" != "$COREPACK_VERSION" ]; then + echo "Error: corepack version mismatch: expected $COREPACK_VERSION, got '$installed_corepack'." + echo "The bundled Corepack on PATH may be taking precedence over the npm-global install." + echo "Ensure your npm global bin directory comes before any other Node.js install on PATH," + echo "or use a Node version manager (nvm, asdf, fnm) that places the npm prefix appropriately." + exit 1 +fi + +echo "" +echo "Enabling Corepack package manager shims..." +corepack enable + +echo "" +echo "Preparing Yarn from packageManager pin in package.json..." +node ./scripts/prepareCorepackYarn.mjs + echo "" echo "Running yarn install..." -corepack yarn@1.22.22 install --frozen-lockfile --non-interactive +corepack yarn install --frozen-lockfile --non-interactive echo "" echo "Running yarn compile..." -corepack yarn@1.22.22 compile +corepack yarn compile echo "" echo "Building Aspire CLI..." diff --git a/extension/package.json b/extension/package.json index e0463e5a287..45ac656c895 100644 --- a/extension/package.json +++ b/extension/package.json @@ -13,6 +13,7 @@ "engines": { "vscode": "^1.98.0" }, + "packageManager": "yarn@1.22.22", "categories": [ "Programming Languages", "Debuggers", diff --git a/extension/scripts/corepack-version.txt b/extension/scripts/corepack-version.txt new file mode 100644 index 00000000000..6e5caff5f25 --- /dev/null +++ b/extension/scripts/corepack-version.txt @@ -0,0 +1 @@ +0.34.7 diff --git a/extension/scripts/prepareCorepackYarn.mjs b/extension/scripts/prepareCorepackYarn.mjs new file mode 100644 index 00000000000..4660862f9a0 --- /dev/null +++ b/extension/scripts/prepareCorepackYarn.mjs @@ -0,0 +1,215 @@ +#!/usr/bin/env node + +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { homedir, tmpdir } from 'node:os'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import process from 'node:process'; + +const DefaultNpmRegistry = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/'; +// Yarn 1.x `packageManager` strings can carry an integrity suffix when written +// by `corepack use yarn@` (the workflow CONTRIBUTING.MD points contributors +// at to update the pin), producing values like +// "yarn@1.22.22+sha512.f7062e6a5ee1f3aa…". +// The suffix is optional but its presence must not break us. Match an optional +// `+` suffix and ignore it; only the version is needed to seed the cache. +// +// Integrity note: when Corepack's `installVersion` finds an existing cache dir +// containing a `.corepack` file, it returns the recorded `hash`/`bin` +// immediately without re-hashing or signature-checking the install (see +// https://github.com/nodejs/corepack/blob/v0.34.7/sources/corepackUtils.ts). +// Its `Mismatch hashes` check fires only on the download path, which a +// pre-seeded cache never reaches. That means the `sha1.${packEntry.shasum}` we +// write to `.corepack` below is recorded for completeness but is never +// re-verified on reuse — trust on the seeded Yarn rests entirely on the +// `npm pack` fetch from `$NPM_REGISTRY` (the internal dnceng feed), which is +// the same authentication and integrity boundary npm uses for any other +// install through this feed. +// Spec: https://nodejs.org/api/packages.html#packagemanager +const PackageManagerPattern = /^yarn@(?\d+\.\d+\.\d+)(?:\+[\w.-]+)?$/; + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const extensionDirectory = dirname(scriptDirectory); +const packageJsonPath = join(extensionDirectory, 'package.json'); + +const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); +const packageManager = packageJson.packageManager; +const match = typeof packageManager === 'string' ? PackageManagerPattern.exec(packageManager) : null; + +if (match?.groups?.version === undefined) { + fail(`Expected packageManager in ${packageJsonPath} to be an exact Yarn Classic version like "yarn@1.22.22", but found ${JSON.stringify(packageManager)}.`); +} + +const yarnVersion = match.groups.version; +const majorVersion = Number(yarnVersion.split('.')[0]); + +if (majorVersion >= 2) { + fail(`The Corepack cache seeding workaround only supports Yarn Classic (<2.0.0), but packageManager is ${packageManager}. Remove this workaround and use Corepack's native prepare/install flow for Yarn Berry.`); +} + +const registry = process.env.NPM_REGISTRY || DefaultNpmRegistry; +const corepackHome = getCorepackHome(); +const installDirectory = join(corepackHome, 'v1', 'yarn', yarnVersion); +const installParentDirectory = dirname(installDirectory); +const corepackMetadataPath = join(installDirectory, '.corepack'); + +if (existsSync(corepackMetadataPath)) { + console.log(`Corepack cache already contains yarn@${yarnVersion} at ${installDirectory}`); + process.exit(0); +} + +// Race-safe cleanup: only remove the install directory when it is stale +// (no `.corepack` metadata). The earlier existsSync at line 57 races against +// a concurrent winner whose renameSync could complete between that check and +// here; an unconditional rmSync would destroy that just-seeded cache. A +// narrow residual window remains between this re-check and the rmSync, but +// the rename at line 101 below would then fail with EEXIST/ENOTEMPTY and +// fall through to the existing concurrent-cache recovery path. +if (existsSync(installDirectory) && !existsSync(corepackMetadataPath)) { + rmSync(installDirectory, { recursive: true, force: true }); +} + +const temporaryDirectory = mkdtempSync(join(tmpdir(), 'aspire-corepack-yarn-')); +mkdirSync(installParentDirectory, { recursive: true }); +const stagingDirectory = mkdtempSync(join(installParentDirectory, `.yarn-${yarnVersion}-`)); +let cacheSeeded = false; + +try { + console.log(`Packing yarn@${yarnVersion} from ${registry}`); + const npm = getNpmInvocation(); + const packResult = run(npm.command, [...npm.args, 'pack', '--json', '--registry', registry, `yarn@${yarnVersion}`], temporaryDirectory); + const packEntries = parseNpmPackJson(packResult.stdout); + const packEntry = packEntries[0]; + + if (packEntry === undefined || typeof packEntry.filename !== 'string' || typeof packEntry.shasum !== 'string') { + fail(`npm pack did not return the expected filename and shasum metadata. Output: ${packResult.stdout}`); + } + + const tarballPath = join(temporaryDirectory, packEntry.filename); + + // Corepack can use COREPACK_NPM_REGISTRY for npmjs.org, but Azure Artifacts + // does not implement the // metadata route Corepack calls. + // Seed the same cache shape Corepack writes, using npm pack because npm can + // resolve Yarn through the Azure Artifacts pull-through feed. + run('tar', ['-xzf', tarballPath, '-C', stagingDirectory, '--strip-components=1'], temporaryDirectory); + + writeFileSync(join(stagingDirectory, '.corepack'), JSON.stringify({ + locator: { + name: 'yarn', + reference: yarnVersion + }, + bin: { + yarn: './bin/yarn.js', + yarnpkg: './bin/yarn.js' + }, + hash: `sha1.${packEntry.shasum}` + })); + + try { + renameSync(stagingDirectory, installDirectory); + cacheSeeded = true; + } catch (error) { + // Lost a race with a concurrent build (same worktree, same COREPACK_HOME). + // The winner's renameSync atomically populated installDirectory; ours then + // fails because the destination already exists. Filesystem-level error + // codes vary: + // - Windows / macOS HFS+: EEXIST when the destination dir exists. + // - Linux / macOS APFS: ENOTEMPTY (rename(2) rejects renaming over a + // non-empty directory; see + // https://man7.org/linux/man-pages/man2/rename.2.html). + // Both mean the cache is already in place, so log and exit cleanly. Any + // other error code is a real failure (permission, ENOSPC, etc.) and is + // re-thrown. + if (error?.code === 'EEXIST' || error?.code === 'ENOTEMPTY') { + console.log(`Corepack cache already contains yarn@${yarnVersion} at ${installDirectory}`); + } else { + throw error; + } + } + + if (cacheSeeded) { + console.log(`Seeded Corepack cache with yarn@${yarnVersion} at ${installDirectory}`); + } +} finally { + rmSync(temporaryDirectory, { recursive: true, force: true }); + rmSync(stagingDirectory, { recursive: true, force: true }); +} + +function getCorepackHome() { + if (process.env.COREPACK_HOME) { + return process.env.COREPACK_HOME; + } + + // build.sh / build.ps1 / the GitHub Actions workflow / the AzDO pipelines all + // set COREPACK_HOME explicitly to a build-scoped directory. This fallback + // exists only for ad-hoc invocations of this script (e.g. local debugging, + // running `node ./scripts/prepareCorepackYarn.mjs` directly). It mirrors + // Corepack 0.34.x's own cache-path resolution so we seed the directory + // Corepack will later read from. + // Source: https://github.com/nodejs/corepack/blob/v0.34.7/sources/folderUtils.ts + const baseDirectory = process.env.XDG_CACHE_HOME + ?? process.env.LOCALAPPDATA + ?? join(homedir(), process.platform === 'win32' ? 'AppData/Local' : '.cache'); + + return join(baseDirectory, 'node', 'corepack'); +} + +function getNpmInvocation() { + if (process.platform !== 'win32') { + return { command: 'npm', args: [] }; + } + + const npmCliPath = join(dirname(process.execPath), 'node_modules', 'npm', 'bin', 'npm-cli.js'); + + if (!existsSync(npmCliPath)) { + fail(`Could not find npm CLI at ${npmCliPath}. Corepack Yarn cache seeding requires the npm CLI that ships with Node.js.`); + } + + // Avoid spawning npm.cmd directly on Windows. Some hosted images reject the + // .cmd shim from child_process.spawnSync with EINVAL, while invoking the + // npm CLI through node.exe avoids shell/cmd parsing entirely. + return { command: process.execPath, args: [npmCliPath] }; +} + +function parseNpmPackJson(stdout) { + const jsonStart = stdout.indexOf('['); + + if (jsonStart === -1) { + fail(`npm pack did not emit JSON output. Output: ${stdout}`); + } + + return JSON.parse(stdout.slice(jsonStart)); +} + +function run(command, args, cwd) { + const result = spawnSync(command, args, { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'] + }); + + if (result.status !== 0) { + if (result.stdout) { + process.stderr.write(result.stdout); + } + + if (result.stderr) { + process.stderr.write(result.stderr); + } + + const errorDetails = result.error ? ` (${result.error.message})` : ''; + fail(`${command} ${args.join(' ')} failed with exit code ${result.status ?? 'unknown'}${errorDetails}.`); + } + + if (result.stderr) { + process.stderr.write(result.stderr); + } + + return result; +} + +function fail(message) { + console.error(message); + process.exit(1); +}