diff --git a/.github/workflows/external-integration.yml b/.github/workflows/external-integration.yml index 472cde4a048..288714fe4a8 100644 --- a/.github/workflows/external-integration.yml +++ b/.github/workflows/external-integration.yml @@ -66,3 +66,47 @@ jobs: - name: E2E Test run: pnpm test:e2e + + azure-rest-api-specs: + name: Azure REST API Specs + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'int:azure-specs') || github.event_name == 'workflow_dispatch' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup + + - name: Install dependencies + run: pnpm install + + - name: Build and pack TypeSpec Azure packages + run: | + pnpm --filter "!@typespec/playground-website" --filter "!@typespec/website" -r build + + pnpm chronus pack --pack-destination ./tgz-packages --exclude standalone + + echo "Created tgz packages:" + ls -la ./tgz-packages/ + + cd packages/tsp-integration + npm link + + - name: Checkout + run: tsp-integration azure-specs --stage checkout + + - name: Patch package.json + run: tsp-integration azure-specs --stage patch --stage install --tgz-dir ./tgz-packages + + - name: Run TypeSpec validation in azure-rest-api-specs + run: tsp-integration azure-specs --stage validate + + - name: Check for git changes + if: success() || failure() # Still run this step even if validation fails to ensure as much information as possible + run: tsp-integration azure-specs --stage validate:clean diff --git a/.typespec-integration/config.yaml b/.typespec-integration/config.yaml new file mode 100644 index 00000000000..3749a462e33 --- /dev/null +++ b/.typespec-integration/config.yaml @@ -0,0 +1,17 @@ +suites: + azure-specs: + repo: https://github.com/Azure/azure-rest-api-specs + branch: typespec-next + pattern: "specification/**/tspconfig.yaml" + entrypoints: + - name: "client.tsp" + options: ["--no-emit"] + - name: "main.tsp" + azure-specs-pr: + repo: https://github.com/Azure/azure-rest-api-specs-pr + branch: typespec-next + pattern: "specification/**/tspconfig.yaml" + entrypoints: + - name: "client.tsp" + options: ["--no-emit"] + - name: "main.tsp" diff --git a/eng/common/config/labels.ts b/eng/common/config/labels.ts index ae8b867b286..83da5105468 100644 --- a/eng/common/config/labels.ts +++ b/eng/common/config/labels.ts @@ -218,6 +218,10 @@ export default defineConfig({ color: "0969da", description: "Good candidate for MQ", }, + "int:azure-specs": { + color: "0e8a16", + description: "Run integration tests against azure-rest-api-specs", + }, }, }, }, diff --git a/packages/tsp-integration/cmd/tsp-integration.js b/packages/tsp-integration/cmd/tsp-integration.js new file mode 100755 index 00000000000..8fb12721835 --- /dev/null +++ b/packages/tsp-integration/cmd/tsp-integration.js @@ -0,0 +1,2 @@ +#!/usr/bin/env node +import "../dist/cli.js"; diff --git a/packages/tsp-integration/package.json b/packages/tsp-integration/package.json new file mode 100644 index 00000000000..ca9948d0c3a --- /dev/null +++ b/packages/tsp-integration/package.json @@ -0,0 +1,44 @@ +{ + "name": "@azure-tools/integration-tester", + "private": true, + "version": "0.1.0", + "type": "module", + "description": "CLI tool for testing typespec package changes against external repos", + "homepage": "https://github.com/microsoft/typespec", + "license": "MIT", + "author": "Microsoft", + "files": [ + "dist" + ], + "bin": { + "tsp-integration": "./cmd/tsp-integration.js" + }, + "repository": "https://github.com/microsoft/typespec.git", + "engines": { + "node": ">=20.0.0" + }, + "scripts": { + "watch": "tsc -p ./tsconfig.build.json --watch", + "build": "tsc -p ./tsconfig.build.json", + "clean": "rimraf dist/ temp/", + "test": "vitest run", + "test:watch": "vitest -w" + }, + "dependencies": { + "@pnpm/workspace.find-packages": "^1000.0.24", + "execa": "^9.6.1", + "globby": "~16.1.0", + "log-symbols": "^7.0.1", + "ora": "^9.0.0", + "pathe": "^2.0.3", + "picocolors": "~1.1.1", + "simple-git": "^3.28.0", + "tar": "^7.5.2", + "yaml": "~2.8.2" + }, + "devDependencies": { + "typescript": "~5.9.2", + "vitest": "^4.0.15" + }, + "bugs": "https://github.com/microsoft/typespec/issues" +} diff --git a/packages/tsp-integration/src/cli.ts b/packages/tsp-integration/src/cli.ts new file mode 100644 index 00000000000..d538871decc --- /dev/null +++ b/packages/tsp-integration/src/cli.ts @@ -0,0 +1,72 @@ +import { readFile } from "node:fs/promises"; +import { parseArgs } from "node:util"; +import { join, resolve } from "pathe"; +import { parse } from "yaml"; +import { runIntegrationTestSuite, Stages, type Stage } from "./run.js"; +import { ValidationFailedError } from "./utils.js"; + +process.on("SIGINT", () => process.exit(0)); + +const args = parseArgs({ + args: process.argv.slice(2), + allowPositionals: true, + options: { + clean: { + type: "boolean", + default: false, + }, + stage: { + type: "string", + multiple: true, + }, + "tgz-dir": { + type: "string", + }, + repo: { + type: "string", + description: "The path to the repository to test. Defaults temp/{suiteName}.", + }, + interactive: { + type: "boolean", + default: false, + short: "i", + description: "Enable interactive mode for validation.", + }, + }, +}); + +const cwd = process.cwd(); +const integrationDir = join(cwd, ".typespec-integration"); +const suiteName = args.positionals[0]; +const config = parse(await readFile(join(integrationDir, "config.yaml"), "utf8")); +const suite = config.suites[suiteName]; +if (suite === undefined) { + throw new Error(`Integration test suite "${suiteName}" not found in config.`); +} + +let stages: Stage[] | undefined = undefined; +if (args.values.stage) { + stages = args.values.stage as Stage[]; + for (const stage of stages) { + if (!Stages.includes(stage)) { + throw new Error( + `Invalid stage "${stage}" specified. Valid stages are: ${Stages.join(", ")}.`, + ); + } + } +} + +const wd = args.values.repo ?? join(integrationDir, "temp", suiteName); +try { + await runIntegrationTestSuite(wd, suiteName, suite, { + clean: args.values.clean, + stages, + tgzDir: args.values["tgz-dir"] && resolve(process.cwd(), args.values["tgz-dir"]), + interactive: args.values.interactive, + }); +} catch (error) { + if (error instanceof ValidationFailedError) { + process.exit(1); + } + throw error; +} diff --git a/packages/tsp-integration/src/config/types.ts b/packages/tsp-integration/src/config/types.ts new file mode 100644 index 00000000000..bf56d8fedc1 --- /dev/null +++ b/packages/tsp-integration/src/config/types.ts @@ -0,0 +1,15 @@ +export interface IntegrationTestsConfig { + suites: Record; +} + +export interface IntegrationTestSuite { + repo: string; + branch: string; + pattern?: string; + entrypoints?: Entrypoint[]; +} + +export interface Entrypoint { + name: string; + options?: string[]; +} diff --git a/packages/tsp-integration/src/find-packages.ts b/packages/tsp-integration/src/find-packages.ts new file mode 100644 index 00000000000..e51a2369c00 --- /dev/null +++ b/packages/tsp-integration/src/find-packages.ts @@ -0,0 +1,157 @@ +import { findWorkspacePackagesNoCheck } from "@pnpm/workspace.find-packages"; +import { readdir } from "node:fs/promises"; +import { relative, resolve } from "pathe"; +import pc from "picocolors"; +import * as tar from "tar"; +import { log } from "./utils.js"; + +/** + * Collection of packages indexed by package name. + */ +export interface Packages { + [key: string]: { + /** The package name (e.g., "@typespec/compiler") */ + name: string; + /** Absolute path to the package directory or .tgz file */ + path: string; + }; +} + +/** + * Options for {@link findPackages} + */ +export interface FindPackageOptions { + /** Directory containing PNPM workspace to scan for packages */ + wsDir?: string; + /** Directory containing .tgz artifact files */ + tgzDir?: string; +} + +/** + * Finds packages from either a workspace directory or tgz artifact directory. + * + * @param options - Configuration specifying the source to find packages from + * @returns Promise resolving to a collection of discovered packages + * @throws Error if neither wsDir nor tgzDir is provided + */ +export function findPackages(options: FindPackageOptions): Promise { + if (options.tgzDir) { + return findPackagesFromTgzArtifactDir(options.tgzDir); + } + if (options.wsDir) { + return findPackagesFromWorkspace(options.wsDir); + } else { + throw new Error("Either wsDir or tgzDir must be provided to findPackages"); + } +} + +/** + * Prints a formatted list of discovered packages to the console. + * + * @param packages - Collection of packages to display + */ +export function printPackages(packages: Packages): void { + log("Found packages:"); + for (const [name, pkg] of Object.entries(packages)) { + log(` ${pc.green(name)}: ${pc.cyan(relative(process.cwd(), pkg.path))}`); + } +} + +/** + * Discovers packages from a directory containing .tgz artifact files. + * + * This function scans a directory for .tgz files and extracts package information + * by reading the package.json from within each tar file. + * + * @param tgzDir - Directory containing .tgz artifact files + * @returns Promise resolving to discovered packages with paths pointing to .tgz files + */ +export async function findPackagesFromTgzArtifactDir(tgzDir: string): Promise { + const packages: Packages = {}; + + const items = await readdir(tgzDir, { withFileTypes: true }); + const tgzFiles = items + .filter((item) => item.isFile() && item.name.endsWith(".tgz")) + .map((item) => item.name); + + // Process tar files in parallel + await Promise.all( + tgzFiles.map(async (tgzFile) => { + const fullPath = resolve(tgzDir, tgzFile); + const packageName = await extractPackageNameFromTgzFile(fullPath); + + if (packageName) { + packages[packageName] = { + name: packageName, + path: fullPath, + }; + } + }), + ); + + return packages; +} + +/** + * Extracts the package name by reading package.json from a .tgz file. + * + * This function reads the package.json file from the root of the tar archive + * to get the accurate package name, which is more reliable than parsing filenames. + * + * @param tgzFilePath - Path to the .tgz file + * @returns Promise resolving to the package name, or null if not found + */ +async function extractPackageNameFromTgzFile(tgzFilePath: string): Promise { + try { + let packageJsonContent: string | null = null; + + await tar.t({ + file: tgzFilePath, + // cspell:ignore onentry + onentry: (entry) => { + if (entry.path === "package/package.json") { + entry.on("data", (chunk) => { + if (packageJsonContent === null) { + packageJsonContent = ""; + } + packageJsonContent += chunk.toString(); + }); + } + }, + }); + + if (packageJsonContent) { + const packageJson = JSON.parse(packageJsonContent); + return packageJson.name || null; + } + + return null; + } catch (error) { + throw new Error(`Failed to read package.json from ${tgzFilePath}: ${error}`); + } +} + +/** + * Discovers packages from a PNPM workspace configuration. + * + * This function uses PNPM's workspace discovery to find all packages in a monorepo. + * It filters out private packages and packages without names. + * + * @param root - Root directory of the PNPM workspace + * @returns Promise resolving to discovered packages with paths pointing to package directories + */ +export async function findPackagesFromWorkspace(root: string): Promise { + const pnpmPackages = await findWorkspacePackagesNoCheck(root); + const packages: Packages = {}; + + for (const pkg of pnpmPackages) { + if (!pkg.manifest.name || pkg.manifest.private) continue; + + packages[pkg.manifest.name] = { + name: pkg.manifest.name, + path: pkg.rootDirRealPath, + }; + } + + return packages; +} diff --git a/packages/tsp-integration/src/git.ts b/packages/tsp-integration/src/git.ts new file mode 100644 index 00000000000..7da4d45d320 --- /dev/null +++ b/packages/tsp-integration/src/git.ts @@ -0,0 +1,123 @@ +import { execa } from "execa"; +import { mkdir, rm } from "fs/promises"; +import type { Ora } from "ora"; +import { relative } from "pathe"; +import pc from "picocolors"; +import { ResetMode, simpleGit } from "simple-git"; +import type { IntegrationTestSuite } from "./config/types.js"; +import { action, log, ValidationFailedError } from "./utils.js"; +/** + * Options for ensuring repository state. + */ +export interface EnsureRepoStateOptions { + /** If true, forces a clean clone instead of updating existing repository */ + clean?: boolean; +} +/** + * Ensures the repository is in the correct state by either cloning or updating it. + * + * @param suite - Integration test suite configuration containing repo and branch info + * @param dir - Target directory for the repository + * @param options - Options controlling the operation behavior + */ +export async function ensureRepoState( + { repo, branch }: IntegrationTestSuite, + dir: string, + options: EnsureRepoStateOptions = {}, +): Promise { + await action(`Checkout repo ${pc.cyan(repo)} at branch ${pc.cyan(branch)}`, async (spinner) => { + const shouldUpdate = options.clean ? false : await repoExists(dir); + if (shouldUpdate) { + await updateExistingRepo(spinner, { branch }, dir); + } else { + await cloneRepo(spinner, repo, branch, dir); + } + }); +} + +/** + * Checks if a Git repository exists in the specified directory. + * + * @param dir - Directory to check for Git repository + * @returns true if a Git repository exists, false otherwise + */ +async function repoExists(dir: string): Promise { + try { + await execa("git", ["-C", dir, "rev-parse", "--git-dir"]); + return true; + } catch { + return false; + } +} + +/** + * Clones a Git repository from the specified URL to the target directory. + * Performs a shallow clone with depth 1 to minimize download time and disk usage. + * + * @param spinner - Ora spinner for progress indication + * @param repo - Repository URL to clone from + * @param branch - Branch to clone + * @param dir - Target directory for the cloned repository + */ +async function cloneRepo(spinner: Ora, repo: string, branch: string, dir: string): Promise { + const relativeDir = relative(process.cwd(), dir); + + spinner.text = `Cleaning directory ${pc.cyan(relativeDir)}`; + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + + spinner.text = `Cloning repo ${pc.cyan(repo)} at branch ${pc.cyan(branch)} into ${pc.cyan(relativeDir)}`; + await simpleGit().clone(repo, dir, { "--branch": branch, "--depth": 1 }); +} + +/** + * Updates an existing Git repository to the latest state of the specified branch. + * Performs a complete reset of local changes and pulls the latest commits. + * + * @param spinner - Ora spinner for progress indication + * @param suite - Object containing the branch to checkout + * @param dir - Directory containing the Git repository to update + */ +export async function updateExistingRepo( + spinner: Ora, + { branch }: Pick, + dir: string, +): Promise { + const git = simpleGit(dir); + const baseText = spinner.text; + + spinner.text = `${baseText} - Resetting local changes`; + await git.reset(ResetMode.HARD, ["HEAD"]); + await git.clean("fd"); + + spinner.text = `${baseText} - Fetching latest changes`; + await git.fetch("origin"); + + spinner.text = `${baseText} - Checking out branch ${pc.cyan(branch)}`; + await git.checkout(branch); + + spinner.text = `${baseText} - Pulling latest changes`; + await git.pull("origin", branch); +} + +/** + * Validates that the Git repository has no uncommitted changes. + * Logs the status and diff if changes are detected. + * + * @param dir - Directory containing the Git repository to validate + */ +export async function validateGitClean(dir: string): Promise { + const git = simpleGit(dir); + const result = await git.status(); + + if (result.isClean()) { + log(`${pc.green("✔")} No git changes detected`); + } else { + log(`${pc.red("x")} Git changes detected after validation:`); + log(result); + + const diffResult = await execa("git", ["diff", "--color=always"], { cwd: dir }); + log(diffResult.stdout); + throw new ValidationFailedError(); + } +} diff --git a/packages/tsp-integration/src/keyboard-api.test.ts b/packages/tsp-integration/src/keyboard-api.test.ts new file mode 100644 index 00000000000..ae7fef17cdc --- /dev/null +++ b/packages/tsp-integration/src/keyboard-api.test.ts @@ -0,0 +1,68 @@ +import { Readable, Writable } from "stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerConsoleShortcuts } from "./keyboard-api.js"; +import type { TspRunner } from "./validate.js"; + +let runner: TspRunner; +let stdin: NodeJS.ReadStream; +let stdout: Writable; +let cleanup: () => void; + +beforeEach(() => { + runner = { + rerunAll: vi.fn(), + rerunFailed: vi.fn(), + cancelCurrentRun: vi.fn(), + exit: vi.fn(), + isCancelling: false, + } as any; + stdout = new Writable({ + write(chunk, __, callback) { + callback(); + }, + }); + + stdin = new Readable({ read: () => "" }) as NodeJS.ReadStream; + stdin.isTTY = true; + stdin.setRawMode = () => stdin; + cleanup = registerConsoleShortcuts(runner, stdin, stdout); +}); + +afterEach(() => { + cleanup?.(); +}); + +describe("when no test is running", () => { + it("calls exit when q is pressed", () => { + stdin.emit("data", "q"); + expect(runner.exit).toHaveBeenCalled(); + }); + + it("calls rerunAll when a is pressed", () => { + stdin.emit("data", "a"); + expect(runner.rerunAll).toHaveBeenCalled(); + }); + + it("calls rerunFailed when f is pressed", () => { + stdin.emit("data", "f"); + expect(runner.rerunFailed).toHaveBeenCalled(); + }); +}); + +describe("when tests are running", () => { + beforeEach(() => { + runner.runningPromise = Promise.resolve() as any; + }); + describe("calls cancelCurrentRun when cancel keys are pressed", () => { + it.each(["q", "c", "a", "f", "space", "\x03"])(`%s`, (key) => { + stdin.emit("data", key); + expect(runner.cancelCurrentRun).toHaveBeenCalled(); + }); + }); + + it("does NOT call cancelCurrentRun when other keys are pressed", () => { + stdin.emit("data", "b"); + stdin.emit("data", "d"); + expect(runner.cancelCurrentRun).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/tsp-integration/src/keyboard-api.ts b/packages/tsp-integration/src/keyboard-api.ts new file mode 100644 index 00000000000..cc5ae4103b1 --- /dev/null +++ b/packages/tsp-integration/src/keyboard-api.ts @@ -0,0 +1,80 @@ +import pc from "picocolors"; +import readline from "readline"; +import type { Writable } from "stream"; +import type { TspRunner } from "./validate.js"; + +const keys = [ + [["a", "return"], "rerun all tests"], + ["f", "rerun only failed tests"], + ["q", "quit"], +]; +const cancelKeys = ["space", "c", ...keys.map((key) => key[0]).flat()]; + +export function registerConsoleShortcuts( + ctx: TspRunner, + stdin: NodeJS.ReadStream | undefined = process.stdin, + stdout: NodeJS.WriteStream | Writable = process.stdout, +): () => void { + let rl: readline.Interface | undefined; + + async function keypressHandler(str: string, key: readline.Key) { + // Cancel run and exit when ctrl-c or esc is pressed. + // If cancelling takes long and key is pressed multiple times, exit forcefully. + if (str === "\x03" || str === "\x1B" || (key && key.ctrl && key.name === "c")) { + if (!ctx.isCancelling) { + stdout.write(pc.red("Cancelling test run. Press CTRL+c again to exit forcefully.\n")); + process.exitCode = 130; + + await ctx.cancelCurrentRun(); + } + return ctx.exit(); + } + + const name = key?.name; + + if (ctx.runningPromise) { + if (name && cancelKeys.includes(name)) { + stdout.write(pc.yellow("Cancelling current test run...\n")); + await ctx.cancelCurrentRun(); + } + return; + } + + // quit + if (name === "q") { + return ctx.exit(); + } + // rerun all tests + if (name === "a" || name === "return") { + return ctx.rerunAll(); + } + // rerun only failed tests + if (name === "f") { + return ctx.rerunFailed(); + } + } + + function on() { + off(); + rl = readline.createInterface({ input: stdin, escapeCodeTimeout: 50 }); + readline.emitKeypressEvents(stdin, rl); + if (stdin.isTTY) { + stdin.setRawMode(true); + } + stdin.on("keypress", keypressHandler); + } + + function off() { + rl?.close(); + rl = undefined; + stdin.removeListener("keypress", keypressHandler); + if (stdin.isTTY) { + stdin.setRawMode(false); + } + } + + on(); + return function cleanup(): void { + off(); + }; +} diff --git a/packages/tsp-integration/src/patch-package-json.ts b/packages/tsp-integration/src/patch-package-json.ts new file mode 100644 index 00000000000..74b8f47619c --- /dev/null +++ b/packages/tsp-integration/src/patch-package-json.ts @@ -0,0 +1,50 @@ +import { readFileSync } from "fs"; +import { writeFile } from "fs/promises"; +import { join } from "path"; +import { relative } from "pathe"; +import pc from "picocolors"; +import type { Packages } from "./find-packages.js"; +import { log } from "./utils.js"; + +interface PackageJson { + name?: string; + version?: string; + dependencies?: Record; + devDependencies?: Record; + peerDependencies?: Record; + overrides?: Record; + [key: string]: any; +} + +export async function patchPackageJson(dir: string, packages: Packages) { + const packageJsonPath = join(dir, "package.json"); + + // Read existing package.json + const packageJson: PackageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")); + + // Ensure dependency objects exist + packageJson.dependencies = packageJson.dependencies ?? {}; + packageJson.devDependencies = packageJson.devDependencies ?? {}; + packageJson.peerDependencies = packageJson.peerDependencies ?? {}; + packageJson.overrides = packageJson.overrides ?? {}; + + // Update dependencies to point to tgz files + for (const pkg of Object.values(packages)) { + const packageName = pkg.name; + const relativePath = relative(dir, pkg.path); + const filePath = `file:${relativePath}`; + + for (const depType of ["dependencies", "devDependencies", "peerDependencies"]) { + if (packageJson[depType]?.[packageName]) { + packageJson[depType][packageName] = filePath; + log(`Updated ${pc.magenta(depType)}: ${pc.green(packageName)} -> ${pc.cyan(filePath)}`); + } + } + + // Also set in overrides to ensure all nested dependencies use our version + packageJson.overrides[packageName] = filePath; + } + + // Write updated package.json + await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2)); +} diff --git a/packages/tsp-integration/src/run.ts b/packages/tsp-integration/src/run.ts new file mode 100644 index 00000000000..cc6174983d0 --- /dev/null +++ b/packages/tsp-integration/src/run.ts @@ -0,0 +1,75 @@ +import pc from "picocolors"; +import type { IntegrationTestSuite } from "./config/types.js"; +import { findPackages, printPackages } from "./find-packages.js"; +import { ensureRepoState, validateGitClean } from "./git.js"; +import { patchPackageJson } from "./patch-package-json.js"; +import { TaskRunner } from "./runner.js"; +import { action, execWithSpinner, log, repoRoot } from "./utils.js"; +import { validateSpecs } from "./validate.js"; + +export interface RunIntegrationTestSuiteOptions { + /** Only run specific stages. */ + stages?: Stage[]; + /** Clean the temp directory. By default tries to reuse the repo by reseting and pulling latest changes. */ + clean?: boolean; + /** Directory for .tgz files. If not provided it will get the packages from the repo. */ + tgzDir?: string; + /** Enable interactive mode for validation. */ + interactive?: boolean; +} + +export const Stages = ["checkout", "patch", "install", "validate", "validate:clean"] as const; +export type Stage = (typeof Stages)[number]; + +export async function runIntegrationTestSuite( + wd: string, + suiteName: string, + config: IntegrationTestSuite, + options: RunIntegrationTestSuiteOptions = {}, +): Promise { + const runner = new TaskRunner({ verbose: options.clean, stages: options.stages }); + log( + `Running ${options.stages ? options.stages.map(pc.yellow).join(", ") : "all"} stage${options.stages?.length !== 1 ? "s" : ""}`, + pc.cyan(suiteName), + config, + ); + + await runner.stage("checkout", async () => { + await ensureRepoState(config, wd, { + clean: options.clean, + }); + }); + + await runner.stage("patch", async () => { + const packages = await action("Resolving local package versions", async () => { + const packages = await findPackages( + options.tgzDir ? { tgzDir: options.tgzDir } : { wsDir: repoRoot }, + ); + printPackages(packages); + return packages; + }); + + await action("Patching package.json", async () => { + await patchPackageJson(wd, packages); + }); + }); + + await runner.stage("install", async () => { + await action("Installing dependencies", async (spinner) => { + await execWithSpinner(spinner, "npm", ["install", "--no-package-lock"], { + cwd: wd, + }); + await execWithSpinner(spinner, "git", ["checkout", "--", "package.json"], { + cwd: wd, + }); + }); + }); + + await runner.stage("validate", async () => { + await validateSpecs(runner, wd, config, { interactive: options.interactive }); + }); + + await runner.stage("validate:clean", async () => { + await validateGitClean(wd); + }); +} diff --git a/packages/tsp-integration/src/runner.ts b/packages/tsp-integration/src/runner.ts new file mode 100644 index 00000000000..7d3111e0d8b --- /dev/null +++ b/packages/tsp-integration/src/runner.ts @@ -0,0 +1,47 @@ +/* eslint-disable no-console */ +import pc from "picocolors"; + +export interface TaskRunnerOptions { + readonly verbose?: boolean; + readonly stages?: Stages[]; +} + +export class TaskRunner { + #verbose: boolean; + #stages: Stages[] | undefined; + + constructor(options: TaskRunnerOptions = {}) { + this.#stages = options.stages; + this.#verbose = options.verbose === undefined ? Boolean(process.env.CI) : options.verbose; + } + + async stage(name: Stages, fn: () => Promise): Promise { + if (this.#stages && !this.#stages.includes(name)) { + return; + } + await fn(); + } + + reportTaskWithDetails(status: "pass" | "fail" | "skip", name: string, details: string) { + const statusStr = + status === "pass" ? pc.green("pass") : status === "fail" ? pc.red("fail") : pc.gray("skip"); + const message = `${statusStr} ${name}`; + if (this.#verbose || status === "fail") { + this.group(message, details); + } else { + console.log(message); + } + } + + group(name: string, content: string) { + if (process.env.GITHUB_ACTIONS) { + console.log(`::group::${name}`); + console.log(content); + console.log("::endgroup::"); + } else { + console.group(name); + console.log(content); + console.groupEnd(); + } + } +} diff --git a/packages/tsp-integration/src/utils.ts b/packages/tsp-integration/src/utils.ts new file mode 100644 index 00000000000..46ba385d554 --- /dev/null +++ b/packages/tsp-integration/src/utils.ts @@ -0,0 +1,168 @@ +/* eslint-disable no-console */ +import { spawn, type SpawnOptions } from "child_process"; +import logSymbols from "log-symbols"; +import ora, { type Ora } from "ora"; +import { resolve } from "pathe"; +import pc from "picocolors"; + +export class ValidationFailedError extends Error {} + +export const projectRoot = resolve(import.meta.dirname, ".."); +export const repoRoot = resolve(projectRoot, "../.."); + +export async function execWithSpinner( + spinner: Ora, + command: string, + args: string[], + options: SpawnOptions = {}, +): Promise { + return new Promise((resolve, reject) => { + const subprocess = spawn(command, args, { + stdio: "pipe", + ...options, + }); + + // Handle stdout + subprocess.stdout!.on("data", (data) => { + spinner.clear(); + console.log(data.toString()); + spinner.render(); + }); + + // Handle stderr + subprocess.stderr!.on("data", (data) => { + spinner.clear(); + // Ora seems to swallow the stderr output? + console.error(data.toString()); + spinner.render(); + }); + + subprocess.on("close", (code) => { + if (code !== 0) { + reject(new Error(`Command failed with exit code ${code}`)); + } else { + resolve(); + } + }); + }); +} + +export async function action(message: string, fn: (spinner: Ora) => Promise): Promise { + if (process.stderr.isTTY) { + return dynamicAction(message, fn); + } else { + return staticAction(message, fn); + } +} + +export async function dynamicAction( + message: string, + fn: (spinner: Ora) => Promise, +): Promise { + const oldLog = console.log; + + console.log = (...args: any[]) => { + spinner.clear(); + oldLog(...args); + spinner.render(); + }; + const spinner = ora(message).start(); + try { + const result = await fn(spinner); + spinner.succeed(message); + return result; + } catch (error) { + spinner.fail(message); + throw error; + } finally { + console.log = oldLog; + } +} + +export async function staticAction( + message: string, + fn: (spinner: Ora) => Promise, +): Promise { + const spinner = ora(message).start(); + console.log(`- ${message}`); + try { + const result = await fn(spinner); + console.log(`${pc.red(logSymbols.success)} ${message}`); + + return result; + } catch (error) { + console.log(`${pc.red(logSymbols.error)} ${message}`); + throw error; + } +} + +export function log(...args: any[]) { + console.log(...args); +} + +/** Run tasks with limited concurrency */ +export async function runWithConcurrency( + items: T[], + concurrency: number, + processor: (item: T) => Promise, +): Promise { + if (items.length === 0) { + return []; + } + + const toRun = [...items]; + const results: R[] = []; + let completed = 0; + let running = 0; + + return new Promise((resolve, reject) => { + function runNext() { + if (toRun.length === 0 || running >= concurrency) { + return; + } + + const item = toRun.shift(); + if (!item) { + return; + } + + running++; + processor(item) + .then((result) => { + results.push(result); + completed++; + running--; + + if (completed === items.length) { + resolve(results); + return; + } + + runNext(); + }) + .catch((error) => { + reject(error); + }); + } + + // Start initial batch of tasks up to concurrency limit + for (let i = 0; i < Math.min(concurrency, toRun.length); i++) { + runNext(); + } + }); +} + +export async function waitForUserInput(): Promise { + const readline = await import("readline"); + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + return new Promise((resolve) => { + rl.question("", (answer) => { + rl.close(); + resolve(answer.trim()); + }); + }); +} diff --git a/packages/tsp-integration/src/validate.ts b/packages/tsp-integration/src/validate.ts new file mode 100644 index 00000000000..e83b5a5485b --- /dev/null +++ b/packages/tsp-integration/src/validate.ts @@ -0,0 +1,276 @@ +import { execa } from "execa"; +import { readdir } from "fs/promises"; +import { globby } from "globby"; +import { cpus } from "os"; +import { dirname, join, relative } from "pathe"; +import pc from "picocolors"; +import type { Entrypoint, IntegrationTestSuite } from "./config/types.js"; +import { registerConsoleShortcuts } from "./keyboard-api.js"; +import type { TaskRunner } from "./runner.js"; +import { log, runWithConcurrency, ValidationFailedError } from "./utils.js"; + +// Number of parallel TypeSpec compilations to run +const COMPILATION_CONCURRENCY = cpus().length; + +export interface ValidateSpecsOptions { + interactive?: boolean; +} + +export async function validateSpecs( + runner: TaskRunner, + dir: string, + suite: IntegrationTestSuite, + options: ValidateSpecsOptions = {}, +): Promise { + const tspConfigDirs = await findTspProjects(dir, suite.pattern ?? "**/tspconfig.yaml"); + + if (tspConfigDirs.length === 0) { + log("No tspconfig.yaml files found in specification directory"); + return; + } + + runner.group( + `Found ${pc.yellow(tspConfigDirs.length)} TypeSpec projects`, + tspConfigDirs.map((projectDir) => pc.bold(relative(dir, projectDir))).join("\n"), + ); + + const tspRunner = new TspRunner(runner, dir, suite, tspConfigDirs, options); + await tspRunner.run(); +} + +/** Run */ +export class TspRunner { + /** If the runner is currently cancelling */ + isCancelling = false; + runningPromise: Promise | null = null; + + /** Workspace directory */ + dir: string; + + /** Test suit used for this runner */ + suite: IntegrationTestSuite; + + /** Last set of failing projects */ + #failedProjects: string[] = []; + + #runner: TaskRunner; + #projectDirs: string[]; + #options: ValidateSpecsOptions; + + constructor( + runner: TaskRunner, + dir: string, + suite: IntegrationTestSuite, + tspConfigDirs: string[], + options: ValidateSpecsOptions = {}, + ) { + this.#runner = runner; + this.dir = dir; + this.suite = suite; + this.#options = options; + this.#projectDirs = tspConfigDirs; + } + + async run(): Promise { + if (!this.#options.interactive) { + const result = await this.#exec(this.#projectDirs); + if (result.failureCount > 0) { + throw new ValidationFailedError(); + } + return; + } + registerConsoleShortcuts(this); + await this.rerunAll(); + } + + async #exec(projectsToRun: string[]): Promise { + this.runningPromise = this.#execWorker(projectsToRun); + return await this.runningPromise; + } + async #execWorker(projectsToRun: string[]): Promise { + this.isCancelling = false; + const result = await runValidation(this.#runner, this, projectsToRun); + if (this.#options.interactive) { + log( + `\nPress ${pc.yellow("a")} to rerun all tests, ${pc.yellow("f")} to rerun failed tests, or ${pc.yellow("q")} to quit.`, + ); + } + this.#failedProjects = result.failedProjects; + this.runningPromise = null; + return result; + } + + async cancelCurrentRun(): Promise { + if (this.runningPromise) { + this.isCancelling = true; + await this.runningPromise; + this.isCancelling = false; + } + } + + async rerunFailed(): Promise { + process.stdin.write("\x1Bc"); // Clear console + if (this.#failedProjects.length === 0) { + log(pc.green("No failed projects to rerun.")); + return; + } + log(pc.green(`Rerunning ${pc.yellow(this.#failedProjects.length)} failed project(s)...`)); + await this.#exec(this.#failedProjects); + } + + async rerunAll(): Promise { + process.stdin.write("\x1Bc"); // Clear console + log(pc.green(`Running all ${this.#projectDirs.length} projects...`)); + await this.#exec(this.#projectDirs); + } + + exit(): void { + process.exit(this.#failedProjects.length > 0 ? 1 : 0); + } +} + +export interface BatchRunResult { + readonly successCount: number; + readonly failureCount: number; + readonly skippedCount: number; + readonly failedProjects: string[]; +} +async function runValidation( + runner: TaskRunner, + tspRunner: TspRunner, + projectsToRun: string[], +): Promise { + let successCount = 0; + let failureCount = 0; + let skippedCount = 0; + const failedProjects: string[] = []; + + // Create a processor function that handles the compilation and logging + const processProject = async (projectDir: string) => { + if (tspRunner.isCancelling) { + runner.reportTaskWithDetails("skip", relative(tspRunner.dir, projectDir), "Cancelled"); + return { dir: projectDir, result: { status: "skip", output: "Cancelled" } }; + } + const result = await verifyProject(runner, tspRunner.dir, projectDir, tspRunner.suite); + runner.reportTaskWithDetails(result.status, relative(tspRunner.dir, projectDir), result.output); + return { dir: projectDir, result }; + }; + + // Run compilations in parallel with limited concurrency + const results = await runWithConcurrency(projectsToRun, COMPILATION_CONCURRENCY, processProject); + + // Count successes and failures + for (const { dir, result } of results) { + switch (result.status) { + case "skip": + skippedCount++; + break; + case "pass": + successCount++; + break; + case "fail": + failureCount++; + failedProjects.push(dir); + break; + } + } + + log(`\n=== Summary ===`); + const passed = pc.bold(pc.green(`${successCount} passed`)); + const failed = failureCount > 0 ? pc.bold(pc.red(`${failureCount} failed`)) : undefined; + const skipped = skippedCount > 0 ? pc.bold(pc.gray(`${skippedCount} skipped`)) : undefined; + log( + [passed, failed, skipped].filter(Boolean).join(pc.gray(" | ")), + pc.gray(`(${projectsToRun.length})`), + ); + + if (failureCount > 0) { + log("\nFailed folders:"); + failedProjects.forEach((x) => log(` - ${relative(tspRunner.dir, x)}`)); + } + + return { successCount, failureCount, skippedCount, failedProjects }; +} + +async function findTspProjects(wd: string, pattern: string): Promise { + const result = await globby(pattern, { + cwd: wd, + absolute: true, + }); + return result.map((x) => dirname(x)); +} + +/** Find which entrypoints are available */ +async function findTspEntrypoints( + directory: string, + suite: IntegrationTestSuite, +): Promise { + try { + const entries = await readdir(directory); + return (suite.entrypoints ?? [{ name: "main.tsp" }]).filter((entrypoint) => + entries.includes(entrypoint.name), + ); + } catch (error) { + return []; + } +} + +interface ProjectTestResult { + status: "pass" | "fail" | "skip"; + output: string; +} +async function verifyProject( + runner: TaskRunner, + workspaceDir: string, + dir: string, + suite: IntegrationTestSuite, +): Promise { + const entrypoints = await findTspEntrypoints(dir, suite); + + if (entrypoints.length === 0) { + const result: ProjectTestResult = { + status: "fail", + output: `Project '${dir}' has no valid entrypoints to compile. Checked for: ${suite.entrypoints?.map((e) => e.name).join(", ") ?? "main.tsp"}`, + }; + runner.reportTaskWithDetails("fail", dir, result.output); + return result; + } + + let output = ""; + for (const entrypoint of entrypoints) { + const result = await execTspCompile( + workspaceDir, + join(dir, entrypoint.name), + entrypoint.options, + ); + if (!result.success) { + return { status: "fail", output: result.output }; + } else { + output += result.output; + output += `Entrypoint '${entrypoint.name}' compiled successfully.\n`; + } + } + return { status: "pass", output }; +} + +async function execTspCompile( + directory: string, + file: string, + args: string[] = [], +): Promise<{ success: boolean; output: string }> { + const { failed, all } = await execa( + "npm", + ["exec", "--no", "--", "tsp", "compile", file, "--warn-as-error", ...args], + { + cwd: directory, + stdio: "pipe", + all: true, + reject: false, + env: { FORCE_COLOR: pc.isColorSupported ? "1" : undefined }, // Force color output + }, + ); + return { + success: !failed, + output: all, + }; +} diff --git a/packages/tsp-integration/tsconfig.build.json b/packages/tsp-integration/tsconfig.build.json new file mode 100644 index 00000000000..ab4fd1577ad --- /dev/null +++ b/packages/tsp-integration/tsconfig.build.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "references": [], + "include": ["src"], + "exclude": ["**/*.test.*", "test/**/*"] +} diff --git a/packages/tsp-integration/tsconfig.json b/packages/tsp-integration/tsconfig.json new file mode 100644 index 00000000000..a79ad487af4 --- /dev/null +++ b/packages/tsp-integration/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "verbatimModuleSyntax": true, + "rootDir": "src", + "outDir": "dist" + } +} diff --git a/packages/tsp-integration/vitest.config.ts b/packages/tsp-integration/vitest.config.ts new file mode 100644 index 00000000000..63cad767f57 --- /dev/null +++ b/packages/tsp-integration/vitest.config.ts @@ -0,0 +1,4 @@ +import { defineConfig, mergeConfig } from "vitest/config"; +import { defaultTypeSpecVitestConfig } from "../../vitest.config.js"; + +export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 777b2cef346..29c243f0db6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2269,6 +2269,46 @@ importers: specifier: ~5.9.2 version: 5.9.3 + packages/tsp-integration: + dependencies: + '@pnpm/workspace.find-packages': + specifier: ^1000.0.24 + version: 1000.0.55(@pnpm/logger@1001.0.1)(@pnpm/worker@1000.6.2(@pnpm/logger@1001.0.1)(@types/node@25.0.9))(typanion@3.14.0) + execa: + specifier: ^9.6.1 + version: 9.6.1 + globby: + specifier: ~16.1.0 + version: 16.1.0 + log-symbols: + specifier: ^7.0.1 + version: 7.0.1 + ora: + specifier: ^9.0.0 + version: 9.0.0 + pathe: + specifier: ^2.0.3 + version: 2.0.3 + picocolors: + specifier: ~1.1.1 + version: 1.1.1 + simple-git: + specifier: ^3.28.0 + version: 3.30.0 + tar: + specifier: ^7.5.2 + version: 7.5.4 + yaml: + specifier: ~2.8.2 + version: 2.8.2 + devDependencies: + typescript: + specifier: ~5.9.2 + version: 5.9.3 + vitest: + specifier: ^4.0.15 + version: 4.0.17(@types/node@25.0.9)(@vitest/ui@4.0.17)(happy-dom@20.3.4)(jsdom@25.0.1)(tsx@4.21.0)(yaml@2.8.2) + packages/tspd: dependencies: '@alloy-js/core': @@ -4769,6 +4809,12 @@ packages: peerDependencies: koa: ^2.0.0 || ^3.0.0 + '@kwsites/file-exists@1.1.1': + resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + + '@kwsites/promise-deferred@1.1.1': + resolution: {integrity: sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==} + '@mdx-js/mdx@3.1.1': resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==} @@ -12006,6 +12052,9 @@ packages: simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-git@3.30.0: + resolution: {integrity: sha512-q6lxyDsCmEal/MEGhP1aVyQ3oxnagGlBDOVSIB4XUVLl1iZh0Pah6ebC9V4xBap/RfgP2WlI8EKs0WS0rMEJHg==} + sirv@3.0.2: resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} engines: {node: '>=18'} @@ -16509,6 +16558,14 @@ snapshots: transitivePeerDependencies: - supports-color + '@kwsites/file-exists@1.1.1': + dependencies: + debug: 4.4.3(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + + '@kwsites/promise-deferred@1.1.1': {} + '@mdx-js/mdx@3.1.1': dependencies: '@types/estree': 1.0.8 @@ -26193,6 +26250,14 @@ snapshots: simple-concat: 1.0.1 optional: true + simple-git@3.30.0: + dependencies: + '@kwsites/file-exists': 1.1.1 + '@kwsites/promise-deferred': 1.1.1 + debug: 4.4.3(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + sirv@3.0.2: dependencies: '@polka/url': 1.0.0-next.29