Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Use Node.js v24 as the base image
# Use Node.js v25 as the base image
FROM node:25-alpine

# Set working directory
Expand Down
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
# cql-tests-runner

[![Website](https://shields.foundry.hl7.org/website?url=https%3A%2F%2Fcql-tests-runner.quality.hl7.org&logo=fireship&label=try%20it%20now)](https://cql-tests-runner.quality.hl7.org)
[![GitHub contributors](https://shields.foundry.hl7.org/github/contributors/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner/graphs/contributors)
[![GitHub last commit](https://shields.foundry.hl7.org/github/last-commit/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner/graphs/commit-activity)
[![GitHub top language](https://shields.foundry.hl7.org/github/languages/top/cqframework/cql-tests-runner?logo=github)](https://github.com/cqframework/cql-tests-runner)
[![Docker automated build](https://shields.foundry.hl7.org/docker/automated/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner)
[![Docker pulls](https://shields.foundry.hl7.org/docker/pulls/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner)
[![Docker image size](https://shields.foundry.hl7.org/docker/image-size/hlseven/quality-cql-tests-runner?logo=docker)](https://hub.docker.com/r/hlseven/quality-cql-tests-runner)


Test Runner for the [CQL Tests](https://github.com/cqframework/cql-tests) repository. This node application allows you to run the tests in the CQL Tests repository against a server of your choice using the [$cql](https://hl7.org/fhir/uv/cql/OperationDefinition-cql-cql.html) operation. The runner in its current state uses only this operation, and there is no expectation of any other FHIR server capability made by this runner. Additional capabilities may be required in the future as we expand the runner to support full Library/$evaluate as well. None of the tests in the repository have any expectation of being able to access data (i.e. the tests have no retrieve expressions).

The application runs all the tests in the repository and outputs the results as a JSON file in the `results` directory. If the output directory does not exist, it will be created.
Expand All @@ -8,11 +17,11 @@ Results output from running these tests can be posted to the [CQL Tests Results]

## Setting up the Environment

This application requires Node v24 and makes use of the [Axios](https://axios-http.com/docs/intro) framework for HTTP request/response processing. [Node Download](https://nodejs.org/en/download)
This application requires Node v25 and makes use of the [Axios](https://axios-http.com/docs/intro) framework for HTTP request/response processing. [Node Download](https://nodejs.org/en/download)

Install application dependencies using

```
```sh
npm install
```

Expand Down
309 changes: 77 additions & 232 deletions package-lock.json

Large diffs are not rendered by default.

17 changes: 8 additions & 9 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "cql-tests-runner",
"version": "1.5.0",
"version": "1.6.2",
"description": "Server API and command line tools for running CQL tests",
"type": "module",
"main": "dist/bin/cql-tests.js",
Expand All @@ -23,17 +23,17 @@
"author": "",
"license": "Apache-2.0",
"dependencies": {
"@modelcontextprotocol/sdk": "^1.25.3",
"@modelcontextprotocol/sdk": "^1.27.1",
"@types/uuid": "^11.0.0",
"antlr4": "4.13.2",
"axios": "^1.13.3",
"axios": "^1.13.6",
"colors": "^1.4.0",
"commander": "^14.0.2",
"config": "^4.2.0",
"commander": "^14.0.3",
"config": "^4.4.1",
"cors": "^2.8.6",
"date-fns": "^4.1.0",
"express": "^5.2.1",
"fast-xml-parser": "^5.3.3",
"fast-xml-parser": "^5.4.2",
"uuid": "^13.0.0",
"zod": "^4.3.6",
"zod-from-json-schema": "^0.5.2"
Expand All @@ -42,11 +42,10 @@
"@types/cors": "^2.8.19",
"@types/express": "^5.0.6",
"@types/fhir": "^0.0.41",
"@types/node": "^25.0.10",
"@types/supertest": "^6.0.3",
"@types/node": "^25.3.3",
"@types/supertest": "^7.2.0",
"prettier": "^3.8.1",
"supertest": "^7.2.2",
"ts-node": "^10.9.2",
"tsx": "^4.21.0",
"typescript": "^5.9.3",
"vitest": "^4.0.18"
Expand Down
18 changes: 13 additions & 5 deletions src/cql-engine/cql-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,21 @@ export class CQLEngine {
* Creates an instance of CQLEngine.
* @param baseURL - The base URL for the CQL engine.
* @param cqlPath - The path for the CQL engine (optional).
* @param cqlTranslator - CQL translator name (optional, from config Build).
* @param cqlTranslatorVersion - CQL translator version (optional).
* @param cqlEngine - CQL engine name (optional).
* @param cqlEngineVersion - CQL engine version (optional).
*/
constructor(baseURL: string, cqlPath: string | null = null,
cqlTranslator: string, cqlTranslatorVersion: string,
cqlEngine: string, cqlEngineVersion: string) {
constructor(
baseURL: string,
cqlPath: string | null = null,
cqlTranslator: string = '',
cqlTranslatorVersion: string = '',
cqlEngine: string = '',
cqlEngineVersion: string = ''
) {
this._prepareBaseURL(baseURL, cqlPath);
this._setInformationFields(cqlTranslator, cqlTranslatorVersion,
cqlEngine, cqlEngineVersion);
this._setInformationFields(cqlTranslator, cqlTranslatorVersion, cqlEngine, cqlEngineVersion);
}

/**
Expand Down
6 changes: 5 additions & 1 deletion src/models/config-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ export interface Config {
CqlFileVersion: string;
CqlOutputPath: string;
CqlVersion?: string;
testsRunDescription?: string; // Note: schema has this misplaced but it's used in code
testsRunDescription?: string;
cqlTranslator?: string;
cqlTranslatorVersion?: string;
cqlEngine?: string;
cqlEngineVersion?: string;
};
Tests: {
ResultsPath: string;
Expand Down
172 changes: 70 additions & 102 deletions src/server/test-execution-service.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// Author: Preston Lee

import { ConfigLoader } from '../conf/config-loader.js';
import { CQLEngine } from '../cql-engine/cql-engine.js';
import { CQLTestResults } from '../test-results/cql-test-results.js';
Expand All @@ -7,17 +9,55 @@ import {
generateParametersResource,
Result,
} from '../shared/results-shared.js';
import { InternalTestResult } from '../models/test-types.js';
import { InternalTestResult, Tests } from '../models/test-types.js';
import { ServerConnectivity } from '../shared/server-connectivity.js';
import { ResultExtractor } from '../extractors/result-extractor.js';
import { buildExtractor } from './extractor-builder.js';
import { createConfigFromData } from './config-utils.js';
import { resultsEqual } from '../shared/results-utils.js';

// Type declaration for CVL loader
declare const cvlLoader: () => Promise<[{ default: any }]>;
interface ExecutionContext {
config: ConfigLoader;
cqlEngine: CQLEngine;
cvl: any;
tests: Tests[];
resultExtractor: ResultExtractor;
skipMap: Map<string, string>;
}

export class TestExecutionService {
/**
* Builds shared execution context from config data (engine, CVL, tests, extractor, skip map).
*/
private async createExecutionContext(configData: any): Promise<ExecutionContext> {
const config = createConfigFromData(configData);
const serverBaseUrl = config.FhirServer.BaseUrl;
const cqlEndpoint = config.CqlEndpoint;

await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);

const build = config.Build;
const cqlEngine = new CQLEngine(
serverBaseUrl,
cqlEndpoint,
build?.cqlTranslator ?? '',
build?.cqlTranslatorVersion ?? '',
build?.cqlEngine ?? '',
build?.cqlEngineVersion ?? ''
);
cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5';

// @ts-expect-error - cvl.mjs has no declaration file
const cvlModule = await import('../../cvl/cvl.mjs');
const cvl = cvlModule.default;

const tests = TestLoader.load();
const resultExtractor = buildExtractor();
const skipMap = config.skipListMap();

return { config, cqlEngine, cvl, tests, resultExtractor, skipMap };
}

/**
* Runs a single test
*/
Expand Down Expand Up @@ -86,28 +126,11 @@ export class TestExecutionService {
* Runs all tests based on configuration
*/
async runTests(configData: any): Promise<any> {
// Create a temporary config loader from the provided data
const config = createConfigFromData(configData);
const serverBaseUrl = config.FhirServer.BaseUrl;
const cqlEndpoint = config.CqlEndpoint;
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap } = ctx;

// Verify server connectivity before proceeding
await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);

const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint);
cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5';

// Load CVL using dynamic import
// @ts-ignore
const cvlModule = await import('../../cvl/cvl.mjs');
const cvl = cvlModule.default;

const tests = TestLoader.load();
const quickTest = config.Debug?.QuickTest || false;
const resultExtractor = buildExtractor();
const emptyResults = await generateEmptyResults(tests, quickTest);
const skipMap = config.skipListMap();

const results = new CQLTestResults(cqlEngine);

for (const testFile of emptyResults) {
Expand All @@ -117,7 +140,6 @@ export class TestExecutionService {
}
}

// Return the results data that would normally be written to file
return results.toJSON();
}

Expand All @@ -130,50 +152,22 @@ export class TestExecutionService {
testName: string,
configData: any
): Promise<any> {
const config = createConfigFromData(configData);
const serverBaseUrl = config.FhirServer.BaseUrl;
const cqlEndpoint = config.CqlEndpoint;

await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap } = ctx;

const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint);
cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5';

// @ts-ignore
const cvlModule = await import('../../cvl/cvl.mjs');
const cvl = cvlModule.default;

const tests = TestLoader.load();
const resultExtractor = buildExtractor();
const skipMap = config.skipListMap();

// Find the specific test
for (const testSuite of tests) {
if (testSuite.name === testsName) {
for (const group of testSuite.group) {
if (group.name === groupName && group.test) {
for (const test of group.test) {
if (test.name === testName) {
const result = new Result(testsName, groupName, test);
await this.runTest(
result,
cqlEngine.apiUrl!,
cvl,
resultExtractor,
skipMap,
config
);

// Convert to schema-compliant format
const testResults = new CQLTestResults(cqlEngine);
testResults.add(result);
const jsonResults = testResults.toJSON();

// Return just the single test result
return jsonResults.results[0] || null;
}
}
}
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
if (test.name !== testName) continue;

const result = new Result(testsName, groupName, test);
await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, config);

const testResults = new CQLTestResults(cqlEngine);
testResults.add(result);
return testResults.toJSON().results[0] ?? null;
}
}
}
Expand All @@ -189,50 +183,24 @@ export class TestExecutionService {
groupName: string,
configData: any
): Promise<any[]> {
const config = createConfigFromData(configData);
const serverBaseUrl = config.FhirServer.BaseUrl;
const cqlEndpoint = config.CqlEndpoint;

await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);

const cqlEngine = new CQLEngine(serverBaseUrl, cqlEndpoint);
cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5';

// @ts-ignore
const cvlModule = await import('../../cvl/cvl.mjs');
const cvl = cvlModule.default;

const tests = TestLoader.load();
const resultExtractor = buildExtractor();
const skipMap = config.skipListMap();
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap } = ctx;

const results = new CQLTestResults(cqlEngine);

// Find and run tests in the specified group
for (const testSuite of tests) {
if (testSuite.name === testsName) {
for (const group of testSuite.group) {
if (group.name === groupName && group.test) {
for (const test of group.test) {
const result = new Result(testsName, groupName, test);
await this.runTest(
result,
cqlEngine.apiUrl!,
cvl,
resultExtractor,
skipMap,
config
);
results.add(result);
}
break;
}
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
const result = new Result(testsName, groupName, test);
await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, config);
results.add(result);
}
break;
return results.toJSON().results;
}
}

const jsonResults = results.toJSON();
return jsonResults.results;
return results.toJSON().results;
}
}
15 changes: 8 additions & 7 deletions src/services/test-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,15 @@ export class TestRunner {
// Verify server connectivity before proceeding
await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);

const build = config.Build;
const cqlEngine = new CQLEngine(
serverBaseUrl,
cqlEndpoint,
configData.Build.cqlTranslator,
configData.Build.cqlTranslatorVersion,
configData.Build.cqlEngine,
configData.Build.cqlEngineVersion
);
serverBaseUrl,
cqlEndpoint,
build.cqlTranslator ?? '',
build.cqlTranslatorVersion ?? '',
build.cqlEngine ?? '',
build.cqlEngineVersion ?? ''
);
cqlEngine.cqlVersion = '1.5'; //default value
const cqlVersion = config.Build?.CqlVersion;
if (typeof cqlVersion === 'string' && cqlVersion.trim() !== '') {
Expand Down