[issues/562] Add CI job for integration tests with marketplace extensions

.github/actions/run-integration-tests-with-extensions/action.yml

@@ -0,0 +1,22 @@
+name: 'Run Integration Tests (with extensions)'
+description: 'Run pnpm test:release:with-extensions --no-assisted under Xvfb — installs marketplace extensions and runs automated tests that depend on them'
+runs:
+  using: 'composite'
+  steps:
+    - name: Install Xvfb and VS Code native dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        ALSA_PKG=$(apt-cache show libasound2t64 >/dev/null 2>&1 && echo libasound2t64 || echo libasound2)
+        sudo apt-get install -y --no-install-recommends xvfb libgtk-3-0 libxss1 libnss3 "$ALSA_PKG"
+
+    - name: Cache VS Code test binary
+      uses: actions/cache@v4
+      with:
+        path: ~/.vscode-test
+        key: vscode-test-${{ runner.os }}-stable
+        restore-keys: vscode-test-${{ runner.os }}-
+
+    - name: Run integration tests under Xvfb
+      shell: bash
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pnpm test:release:with-extensions --no-assisted

.github/workflows/ci.yml

@@ -42,3 +42,20 @@ jobs:
           # For PRs: compare with base branch
           # For pushes to main: just count current TODOs
           base-ref: ${{ github.event_name == 'pull_request' && github.base_ref || '' }}
+
+  test-with-extensions:
+    name: Integration Tests (with extensions)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js and pnpm
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Install dependencies
+        uses: ./.github/actions/install-deps
+
+      - name: Run integration tests with extensions
+        uses: ./.github/actions/run-integration-tests-with-extensions

packages/rangelink-vscode-extension/.vscode-test.automated.mjs

@@ -1,26 +1,10 @@
-import * as os from 'node:os';
-import * as path from 'node:path';
-
 import { defineConfig } from '@vscode/test-cli';
 
-const MOCHA_TIMEOUT_MS = 20_000;
-const ASSISTED_TEST_GREP = '\\[assisted\\]';
-const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test');
+import { CI_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs';
 
 export default defineConfig([
   {
-    files: 'out/__integration-tests__/suite/**/*.test.js',
-    extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'],
-    workspaceFolder: './',
-    version: 'stable',
-    launchArgs: ['--user-data-dir', USER_DATA_DIR],
-    env: {
-      RANGELINK_CAPTURE_LOGS: 'true',
-    },
-    mocha: {
-      timeout: MOCHA_TIMEOUT_MS,
-      grep: ASSISTED_TEST_GREP,
-      invert: true,
-    },
+    ...BASE_CONFIG,
+    mocha: { timeout: CI_TIMEOUT_MS, ...BASE_CONFIG.mocha },
   },
 ]);

packages/rangelink-vscode-extension/.vscode-test.base.mjs

@@ -0,0 +1,29 @@
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+// 10 minutes per test — assisted tests block on human interaction.
+export const ASSISTED_TIMEOUT_MS = 600_000;
+
+// No human in CI — automated tests resolve in under 5s.
+export const CI_TIMEOUT_MS = 20_000;
+
+export const userDataDir = (suffix = '') => [
+  '--user-data-dir',
+  path.join(os.tmpdir(), `rl-vscode-test${suffix}`),
+];
+
+// grep and invert are driven by env vars set in test-release-run.sh.
+const envMocha = () => ({
+  ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}),
+  ...(process.env.MOCHA_INVERT === 'true' ? { invert: true } : {}),
+});
+
+export const BASE_CONFIG = {
+  files: 'out/__integration-tests__/suite/**/*.test.js',
+  extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'],
+  workspaceFolder: './',
+  version: 'stable',
+  launchArgs: userDataDir(),
+  env: { RANGELINK_CAPTURE_LOGS: 'true' },
+  mocha: envMocha(),
+};

packages/rangelink-vscode-extension/.vscode-test.mjs

@@ -1,28 +1,10 @@
-import * as os from 'node:os';
-import * as path from 'node:path';
-
 import { defineConfig } from '@vscode/test-cli';
 
-// 10 minutes per test — assisted tests block on human interaction (modal
-// verdict dialogs don't auto-dismiss), so the human needs headroom to read
-// instructions, complete UI actions, or step away for a break. Automated tests
-// never come close to this threshold.
-const MOCHA_TIMEOUT_MS = 600_000;
-const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test');
+import { ASSISTED_TIMEOUT_MS, BASE_CONFIG } from './.vscode-test.base.mjs';
 
 export default defineConfig([
   {
-    files: 'out/__integration-tests__/suite/**/*.test.js',
-    extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'],
-    workspaceFolder: './',
-    version: 'stable',
-    launchArgs: ['--user-data-dir', USER_DATA_DIR],
-    env: {
-      RANGELINK_CAPTURE_LOGS: 'true',
-    },
-    mocha: {
-      timeout: MOCHA_TIMEOUT_MS,
-      ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}),
-    },
+    ...BASE_CONFIG,
+    mocha: { timeout: ASSISTED_TIMEOUT_MS, ...BASE_CONFIG.mocha },
   },
 ]);

packages/rangelink-vscode-extension/.vscode-test.with-extensions.mjs

@@ -1,31 +1,17 @@
-import * as os from 'node:os';
-import * as path from 'node:path';
-
 import { defineConfig } from '@vscode/test-cli';
 
-// Same timeout as the main config — assisted tests block on human interaction.
-const MOCHA_TIMEOUT_MS = 600_000;
-const USER_DATA_DIR = path.join(os.tmpdir(), 'rl-vscode-test-with-ext');
+import { ASSISTED_TIMEOUT_MS, BASE_CONFIG, userDataDir } from './.vscode-test.base.mjs';
 
-// Extensions installed from the marketplace before tests run.
-// With these present, isClaudeCodeAvailable() returns true, enabling [assisted]
-// tests that verify real focus + paste behavior.
+// Marketplace extensions installed before tests run. With these present,
+// isClaudeCodeAvailable() returns true, enabling tests that verify real
+// focus + paste behavior.
 const MARKETPLACE_EXTENSIONS = ['anthropic.claude-code'];
 
 export default defineConfig([
   {
-    files: 'out/__integration-tests__/suite/**/*.test.js',
-    extensionDevelopmentPath: ['./', './test-fixtures/dummy-ai-extension/'],
-    workspaceFolder: './',
-    version: 'stable',
+    ...BASE_CONFIG,
+    launchArgs: userDataDir('-with-ext'),
     installExtensions: MARKETPLACE_EXTENSIONS,
-    launchArgs: ['--user-data-dir', USER_DATA_DIR],
-    env: {
-      RANGELINK_CAPTURE_LOGS: 'true',
-    },
-    mocha: {
-      timeout: MOCHA_TIMEOUT_MS,
-      ...(process.env.MOCHA_GREP ? { grep: process.env.MOCHA_GREP } : {}),
-    },
+    mocha: { timeout: ASSISTED_TIMEOUT_MS, ...BASE_CONFIG.mocha },
   },
 ]);

packages/rangelink-vscode-extension/TESTING.md

@@ -13,7 +13,7 @@
 | Coverage report          | `pnpm test:coverage` (from extension dir)                     | Before PR / on demand                                                                     | ✅ (with thresholds) |
 | Integration tests        | `pnpm test:release`                                           | Before PR, after feature work                                                             | —                    |
 | Integration (CI-safe)    | `pnpm test:release:automated`                                 | CI / headless environments                                                                | ✅                   |
-| Integration (extensions) | `pnpm test:release:with-extensions`                           | Tests needing real AI extensions                                                          | —                    |
+| Integration (extensions) | `pnpm test:release:with-extensions`                           | Tests needing real AI extensions                                                          | ✅                   |
 | Integration (filter)     | `pnpm test:release:grep "<pattern>"`                          | Run specific TCs by ID or suite                                                           | —                    |
 | Prepare QA test plan     | `pnpm generate:qa-test-plan:vscode-extension`                 | Start of release cycle                                                                    | —                    |
 | Generate QA issue        | `pnpm generate:qa-issue:vscode-extension`                     | At the start of each release cycle                                                        | —                    |

packages/rangelink-vscode-extension/scripts/setup-integration-test-settings.js

@@ -13,7 +13,14 @@ const fs = require('node:fs');
 const os = require('node:os');
 const path = require('node:path');
 
-const SETTINGS_DIR = path.join(os.tmpdir(), 'rl-vscode-test', 'User');
+const suffixIndex = process.argv.indexOf('--suffix');
+if (suffixIndex !== -1 && !process.argv[suffixIndex + 1]) {
+  console.error('Error: --suffix requires a value');
+  process.exit(1);
+}
+const suffix = suffixIndex !== -1 ? process.argv[suffixIndex + 1] : '';
+const DATA_DIR = `rl-vscode-test${suffix}`;
+const SETTINGS_DIR = path.join(os.tmpdir(), DATA_DIR, 'User');
 const SETTINGS_FILE = path.join(SETTINGS_DIR, 'settings.json');
 
 const settings = {

packages/rangelink-vscode-extension/scripts/test-release-run.sh

@@ -105,6 +105,11 @@ if [[ -n "$LABEL_FILTER" ]]; then
   fi
 fi
 
+if [[ -z "$GREP_PATTERN" && ( "$MODE" == "automated" || "$NO_ASSISTED" == true ) ]]; then
+  export MOCHA_GREP='\[assisted\]'
+  export MOCHA_INVERT=true
+fi
+
 OUTPUT_DIR="$PACKAGE_ROOT/qa/output"
 mkdir -p "$OUTPUT_DIR"
 TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S")
@@ -139,9 +144,16 @@ echo ""
 
 pnpm test:release:prepare
 
+if [[ "$WITH_EXTENSIONS" == "true" ]]; then
+  node "$SCRIPT_DIR/setup-integration-test-settings.js" --suffix -with-ext
+fi
+
 TEST_EXIT=0
 # shellcheck disable=SC2086
-MOCHA_GREP="$GREP_PATTERN" npx vscode-test $VSCODE_TEST_CONFIG 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | tee -a "$REPORT_FILE" || TEST_EXIT=$?
+if [[ -n "$GREP_PATTERN" ]]; then
+  export MOCHA_GREP="$GREP_PATTERN"
+fi
+npx vscode-test $VSCODE_TEST_CONFIG 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | tee -a "$REPORT_FILE" || TEST_EXIT=$?
 
 if [[ -n "$GREP_PATTERN" && $TEST_EXIT -eq 0 ]]; then
   if ! grep -qE '[1-9][0-9]* passing' "$REPORT_FILE"; then