diff --git a/.github/actions/run-integration-tests-with-extensions/action.yml b/.github/actions/run-integration-tests-with-extensions/action.yml new file mode 100644 index 00000000..f1776886 --- /dev/null +++ b/.github/actions/run-integration-tests-with-extensions/action.yml @@ -0,0 +1,22 @@ +name: 'Run Integration Tests (with extensions)' +description: 'Run pnpm test:release:with-extensions --no-assisted under Xvfb — installs marketplace extensions and runs automated tests that depend on them' +runs: + using: 'composite' + steps: + - name: Install Xvfb and VS Code native dependencies + shell: bash + run: | + sudo apt-get update + ALSA_PKG=$(apt-cache show libasound2t64 >/dev/null 2>&1 && echo libasound2t64 || echo libasound2) + sudo apt-get install -y --no-install-recommends xvfb libgtk-3-0 libxss1 libnss3 "$ALSA_PKG" + + - name: Cache VS Code test binary + uses: actions/cache@v4 + with: + path: ~/.vscode-test + key: vscode-test-${{ runner.os }}-stable + restore-keys: vscode-test-${{ runner.os }}- + + - name: Run integration tests under Xvfb + shell: bash + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pnpm test:release:with-extensions --no-assisted diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea5220b5..4cd1939a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,3 +42,20 @@ jobs: # For PRs: compare with base branch # For pushes to main: just count current TODOs base-ref: ${{ github.event_name == 'pull_request' && github.base_ref || '' }} + + test-with-extensions: + name: Integration Tests (with extensions) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + + - name: Install dependencies + uses: ./.github/actions/install-deps + + - name: Run integration tests with extensions + uses: ./.github/actions/run-integration-tests-with-extensions diff --git a/packages/rangelink-vscode-extension/.vscode-test.automated.mjs b/packages/rangelink-vscode-extension/.vscode-test.automated.mjs index 7f499d65..b28e32dc 100644 --- a/packages/rangelink-vscode-extension/.vscode-test.automated.mjs +++ b/packages/rangelink-vscode-extension/.vscode-test.automated.mjs @@ -1,26 +1,10 @@ -import * as os from 'node:os'; -import * as path from 'node:path'; - import { defineConfig } from '@vscode/test-cli'; -const MOCHA_TIMEOUT_MS = 20_000; -const ASSISTED_TEST_GREP = '\\[assisted\\]'; -const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test'); +import { CI_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs'; export default defineConfig([ { - files: 'out/__integration-tests__/suite/**/*.test.js', - extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'], - workspaceFolder: './', - version: 'stable', - launchArgs: ['--user-data-dir', USER_DATA_DIR], - env: { - RANGELINK_CAPTURE_LOGS: 'true', - }, - mocha: { - timeout: MOCHA_TIMEOUT_MS, - grep: ASSISTED_TEST_GREP, - invert: true, - }, + ...BASE_CONFIG, + mocha: { timeout: CI_TIMEOUT_MS, ...BASE_CONFIG.mocha }, }, ]); diff --git a/packages/rangelink-vscode-extension/.vscode-test.base.mjs b/packages/rangelink-vscode-extension/.vscode-test.base.mjs new file mode 100644 index 00000000..5826764f --- /dev/null +++ b/packages/rangelink-vscode-extension/.vscode-test.base.mjs @@ -0,0 +1,29 @@ +import * as os from 'node:os'; +import * as path from 'node:path'; + +// 10 minutes per test — assisted tests block on human interaction. +export const ASSISTED_TIMEOUT_MS = 600_000; + +// No human in CI — automated tests resolve in under 5s. +export const CI_TIMEOUT_MS = 20_000; + +export const userDataDir = (suffix = '') => [ + '--user-data-dir', + path.join(os.tmpdir(), `rl-vscode-test${suffix}`), +]; + +// grep and invert are driven by env vars set in test-release-run.sh. +const envMocha = () => ({ + ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}), + ...(process.env.MOCHA_INVERT === 'true' ? { invert: true } : {}), +}); + +export const BASE_CONFIG = { + files: 'out/__integration-tests__/suite/**/*.test.js', + extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'], + workspaceFolder: './', + version: 'stable', + launchArgs: userDataDir(), + env: { RANGELINK_CAPTURE_LOGS: 'true' }, + mocha: envMocha(), +}; diff --git a/packages/rangelink-vscode-extension/.vscode-test.mjs b/packages/rangelink-vscode-extension/.vscode-test.mjs index e4573025..faaf9f7e 100644 --- a/packages/rangelink-vscode-extension/.vscode-test.mjs +++ b/packages/rangelink-vscode-extension/.vscode-test.mjs @@ -1,28 +1,10 @@ -import * as os from 'node:os'; -import * as path from 'node:path'; - import { defineConfig } from '@vscode/test-cli'; -// 10 minutes per test — assisted tests block on human interaction (modal -// verdict dialogs don't auto-dismiss), so the human needs headroom to read -// instructions, complete UI actions, or step away for a break. Automated tests -// never come close to this threshold. -const MOCHA_TIMEOUT_MS = 600_000; -const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test'); +import { ASSISTED_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs'; export default defineConfig([ { - files: 'out/__integration-tests__/suite/**/*.test.js', - extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'], - workspaceFolder: './', - version: 'stable', - launchArgs: ['--user-data-dir', USER_DATA_DIR], - env: { - RANGELINK_CAPTURE_LOGS: 'true', - }, - mocha: { - timeout: MOCHA_TIMEOUT_MS, - ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}), - }, + ...BASE_CONFIG, + mocha: { timeout: ASSISTED_TIMEOUT_MS, ...BASE_CONFIG.mocha }, }, ]); diff --git a/packages/rangelink-vscode-extension/.vscode-test.with-extensions.mjs b/packages/rangelink-vscode-extension/.vscode-test.with-extensions.mjs index 9d7cd2fb..177c2f2a 100644 --- a/packages/rangelink-vscode-extension/.vscode-test.with-extensions.mjs +++ b/packages/rangelink-vscode-extension/.vscode-test.with-extensions.mjs @@ -1,31 +1,17 @@ -import * as os from 'node:os'; -import * as path from 'node:path'; - import { defineConfig } from '@vscode/test-cli'; -// Same timeout as the main config — assisted tests block on human interaction. -const MOCHA_TIMEOUT_MS = 600_000; -const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test-with-ext'); +import { ASSISTED_TIMEOUT_MS, BASE_CONFIG, userDataDir } from './.vscode-test.base.mjs'; -// Extensions installed from the marketplace before tests run. -// With these present, isClaudeCodeAvailable() returns true, enabling [assisted] -// tests that verify real focus + paste behavior. +// Marketplace extensions installed before tests run. With these present, +// isClaudeCodeAvailable() returns true, enabling tests that verify real +// focus + paste behavior. const MARKETPLACE_EXTENSIONS = ['anthropic.claude-code']; export default defineConfig([ { - files: 'out/__integration-tests__/suite/**/*.test.js', - extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'], - workspaceFolder: './', - version: 'stable', + ...BASE_CONFIG, + launchArgs: userDataDir('-with-ext'), installExtensions: MARKETPLACE_EXTENSIONS, - launchArgs: ['--user-data-dir', USER_DATA_DIR], - env: { - RANGELINK_CAPTURE_LOGS: 'true', - }, - mocha: { - timeout: MOCHA_TIMEOUT_MS, - ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}), - }, + mocha: { timeout: ASSISTED_TIMEOUT_MS, ...BASE_CONFIG.mocha }, }, ]); diff --git a/packages/rangelink-vscode-extension/TESTING.md b/packages/rangelink-vscode-extension/TESTING.md index 607a7b76..a85302ea 100644 --- a/packages/rangelink-vscode-extension/TESTING.md +++ b/packages/rangelink-vscode-extension/TESTING.md @@ -13,7 +13,7 @@ | Coverage report | `pnpm test:coverage` (from extension dir) | Before PR / on demand | ✅ (with thresholds) | | Integration tests | `pnpm test:release` | Before PR, after feature work | — | | Integration (CI-safe) | `pnpm test:release:automated` | CI / headless environments | ✅ | -| Integration (extensions) | `pnpm test:release:with-extensions` | Tests needing real AI extensions | — | +| Integration (extensions) | `pnpm test:release:with-extensions` | Tests needing real AI extensions | ✅ | | Integration (filter) | `pnpm test:release:grep ""` | Run specific TCs by ID or suite | — | | Prepare QA test plan | `pnpm generate:qa-test-plan:vscode-extension` | Start of release cycle | — | | Generate QA issue | `pnpm generate:qa-issue:vscode-extension` | At the start of each release cycle | — | diff --git a/packages/rangelink-vscode-extension/scripts/setup-integration-test-settings.js b/packages/rangelink-vscode-extension/scripts/setup-integration-test-settings.js index c9c19a92..cce6e02b 100644 --- a/packages/rangelink-vscode-extension/scripts/setup-integration-test-settings.js +++ b/packages/rangelink-vscode-extension/scripts/setup-integration-test-settings.js @@ -13,7 +13,14 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const SETTINGS_DIR = path.join(os.tmpdir(), 'rl-vscode-test', 'User'); +const suffixIndex = process.argv.indexOf('--suffix'); +if (suffixIndex !== -1 && !process.argv[suffixIndex + 1]) { + console.error('Error: --suffix requires a value'); + process.exit(1); +} +const suffix = suffixIndex !== -1 ? process.argv[suffixIndex + 1] : ''; +const DATA_DIR = `rl-vscode-test${suffix}`; +const SETTINGS_DIR = path.join(os.tmpdir(), DATA_DIR, 'User'); const SETTINGS_FILE = path.join(SETTINGS_DIR, 'settings.json'); const settings = { diff --git a/packages/rangelink-vscode-extension/scripts/test-release-run.sh b/packages/rangelink-vscode-extension/scripts/test-release-run.sh index cc9fa4f9..4182a2d5 100755 --- a/packages/rangelink-vscode-extension/scripts/test-release-run.sh +++ b/packages/rangelink-vscode-extension/scripts/test-release-run.sh @@ -105,6 +105,11 @@ if [[ -n "$LABEL_FILTER" ]]; then fi fi +if [[ -z "$GREP_PATTERN" && ( "$MODE" == "automated" || "$NO_ASSISTED" == true ) ]]; then + export MOCHA_GREP='\[assisted\]' + export MOCHA_INVERT=true +fi + OUTPUT_DIR="$PACKAGE_ROOT/qa/output" mkdir -p "$OUTPUT_DIR" TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S") @@ -139,9 +144,16 @@ echo "" pnpm test:release:prepare +if [[ "$WITH_EXTENSIONS" == "true" ]]; then + node "$SCRIPT_DIR/setup-integration-test-settings.js" --suffix -with-ext +fi + TEST_EXIT=0 # shellcheck disable=SC2086 -MOCHA_GREP="$GREP_PATTERN" npx vscode-test $VSCODE_TEST_CONFIG 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | tee -a "$REPORT_FILE" || TEST_EXIT=$? +if [[ -n "$GREP_PATTERN" ]]; then + export MOCHA_GREP="$GREP_PATTERN" +fi +npx vscode-test $VSCODE_TEST_CONFIG 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | tee -a "$REPORT_FILE" || TEST_EXIT=$? if [[ -n "$GREP_PATTERN" && $TEST_EXIT -eq 0 ]]; then if ! grep -qE '[1-9][0-9]* passing' "$REPORT_FILE"; then