Skip to content

Commit 2eea33f

Browse files
grahamegrievegrahamegrieve
authored
Merge pull request #111 from HealthIntersections/2026-02-gg-background-tasks
fix vsac reloading error, and background task reporting
2 parents 2496367 + b7ba9bc commit 2eea33f

13 files changed

Lines changed: 201 additions & 75 deletions

File tree

‎library/utilities.js‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,58 @@ const Utilities = {
2525
return isNaN(num) ? defaultValue : num;
2626
},
2727
parseFloatOrDefault(value, defaultValue) {
28-
const num = parseFloat(value);
29-
return isNaN(num) ? defaultValue : num;
28+
const num = parseFloat(value);
29+
return isNaN(num) ? defaultValue : num;
3030

3131

32-
}
32+
},
33+
34+
/**
35+
* Format the difference between two Date.now() timestamps for human reading
36+
* @param {number} start - earlier timestamp (from Date.now())
37+
* @param {number} end - later timestamp (from Date.now())
38+
* @returns {string} formatted duration
39+
*/
40+
formatDuration(start, end) {
41+
let ms = Math.abs(end - start);
42+
43+
if (ms < 1000) return `${ms}ms`;
44+
45+
const days = Math.floor(ms / 86400000);
46+
ms %= 86400000;
47+
const hours = Math.floor(ms / 3600000);
48+
ms %= 3600000;
49+
const minutes = Math.floor(ms / 60000);
50+
ms %= 60000;
51+
const seconds = Math.floor(ms / 1000);
52+
ms %= 1000;
53+
54+
const parts = [];
55+
if (days) parts.push(`${days}d`);
56+
if (hours) parts.push(`${hours}h`);
57+
if (minutes) parts.push(`${minutes}m`);
58+
if (seconds || ms) {
59+
parts.push(ms ? `${seconds}.${String(ms).padStart(3, '0')}s` : `${seconds}s`);
60+
}
61+
62+
return parts.join(' ');
63+
},
64+
65+
escapeHtml(text) {
66+
if (typeof text !== 'string') {
67+
return String(text);
68+
}
69+
70+
const map = {
71+
'&': '&amp;',
72+
'<': '&lt;',
73+
'>': '&gt;',
74+
'"': '&quot;',
75+
"'": '&#39;'
76+
};
77+
78+
return text.replace(/[&<>"']/g, function(m) { return map[m]; });
79+
}
3380

3481
};
3582

‎packages/package-crawler.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ const path = require('path');
1313
class PackageCrawler {
1414
log;
1515

16-
constructor(config, db) {
16+
constructor(config, db, stats) {
1717
this.config = config;
1818
this.db = db;
19+
this.stats = stats;
20+
this.stats.task('Package Crawler', 'Initialised');
1921
this.totalBytes = 0;
2022
this.crawlerLog = {};
2123
this.errors = '';
@@ -36,6 +38,7 @@ class PackageCrawler {
3638
};
3739

3840
this.log.info('Running web crawler for packages using master URL: '+ this.config.masterUrl);
41+
this.stats.task('Package Crawler', 'Running');
3942

4043
try {
4144
// Fetch the master JSON file
@@ -54,6 +57,7 @@ class PackageCrawler {
5457
this.log.info('Skipping feed with no URL: '+ feedConfig);
5558
continue;
5659
}
60+
this.stats.task('Package Crawler', 'Running for '+feedConfig.url);
5761

5862
try {
5963
await this.updateTheFeed(
@@ -76,13 +80,15 @@ class PackageCrawler {
7680
this.log.info(`Web crawler completed successfully in ${runTime}ms`);
7781
this.log.info(`Total bytes processed: ${this.totalBytes}`);
7882

83+
this.stats.task('Package Crawler', 'Complete');
7984
return this.crawlerLog;
8085

8186
} catch (error) {
8287
const runTime = Date.now() - startTime;
8388
this.crawlerLog.runTime = `${runTime}ms`;
8489
this.crawlerLog.fatalException = error.message;
8590
this.crawlerLog.endTime = new Date().toISOString();
91+
this.stats.task('Package Crawler', 'Error: '+error.message);
8692

8793
this.log.error('Web crawler failed: '+ error);
8894
throw error;

‎packages/packages.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,7 @@ class PackagesModule {
572572
await this.ensureMirrorDirectory();
573573

574574
// Initialize the crawler
575-
this.crawler = new PackageCrawler(this.config, this.db);
575+
this.crawler = new PackageCrawler(this.config, this.db, this.stats);
576576

577577
// Start the hourly web crawler if enabled
578578
if (config.crawler.enabled) {

‎registry/crawler.js‎

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,16 @@ const MASTER_URL = 'https://fhir.github.io/ig-registry/tx-servers.json';
1414
class RegistryCrawler {
1515
log;
1616

17-
constructor(config = {}) {
17+
constructor(config = {}, stats) {
1818
this.config = {
1919
timeout: config.timeout || 30000, // 30 seconds default
2020
masterUrl: config.masterUrl || MASTER_URL,
2121
userAgent: config.userAgent || 'HealthIntersections/FhirServer',
2222
crawlInterval: config.crawlInterval || 5 * 60 * 1000, // 5 minutes default
2323
apiKeys: config.apiKeys || {} // Map of server URL or code to API key
2424
};
25-
25+
this.stats = stats;
26+
2627
this.currentData = new ServerRegistries();
2728
this.crawlTimer = null;
2829
this.isCrawling = false;
@@ -35,32 +36,32 @@ class RegistryCrawler {
3536
this.log = logv;
3637
}
3738

38-
/**
39-
* Start the crawler with periodic updates
40-
*/
41-
start() {
42-
if (this.crawlTimer) {
43-
return; // Already running
44-
}
45-
46-
// Initial crawl
47-
this.crawl();
48-
49-
// Set up periodic crawling
50-
this.crawlTimer = setInterval(() => {
51-
this.crawl();
52-
}, this.config.crawlInterval);
53-
}
54-
55-
/**
56-
* Stop the crawler
57-
*/
58-
stop() {
59-
if (this.crawlTimer) {
60-
clearInterval(this.crawlTimer);
61-
this.crawlTimer = null;
62-
}
63-
}
39+
// /**
40+
// * Start the crawler with periodic updates
41+
// */
42+
// start() {
43+
// if (this.crawlTimer) {
44+
// return; // Already running
45+
// }
46+
//
47+
// // Initial crawl
48+
// this.crawl();
49+
//
50+
// // Set up periodic crawling
51+
// this.crawlTimer = setInterval(() => {
52+
// this.crawl();
53+
// }, this.config.crawlInterval);
54+
// }
55+
//
56+
// /**
57+
// * Stop the crawler
58+
// */
59+
// stop() {
60+
// if (this.crawlTimer) {
61+
// clearInterval(this.crawlTimer);
62+
// this.crawlTimer = null;
63+
// }
64+
// }
6465

6566
/**
6667
* Main entry point - crawl the registry starting from the master URL
@@ -133,7 +134,8 @@ class RegistryCrawler {
133134
registry.name = registryConfig.name;
134135
registry.authority = registryConfig.authority || '';
135136
registry.address = registryConfig.url;
136-
137+
this.stats.task('TxRegistry', 'Checking: '+registry.address);
138+
137139
if (!registry.name) {
138140
this.addLogEntry('error', 'No name provided for registry', registryConfig.url);
139141
return registry;

‎registry/registry.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class RegistryModule {
2525
this.currentData = null;
2626
this.dataLock = false;
2727
this.stats = stats;
28+
this.stats.task('TxRegistry', 'Initialized');
2829
}
2930

3031
/**
@@ -43,7 +44,7 @@ class RegistryModule {
4344
apiKeys: config.apiKeys || {}
4445
};
4546

46-
this.crawler = new RegistryCrawler(crawlerConfig);
47+
this.crawler = new RegistryCrawler(crawlerConfig, this.stats);
4748
this.crawler.useLog(regLog);
4849

4950
// Initialize API with crawler
@@ -134,6 +135,7 @@ class RegistryModule {
134135
this.logger.info('Crawl already in progress, skipping...');
135136
return;
136137
}
138+
this.stats.task('TxRegistry', 'Crawling');
137139

138140
this.crawlInProgress = true;
139141
this.logger.info('Starting registry crawl...');
@@ -160,9 +162,10 @@ class RegistryModule {
160162
`Found ${newData.registries.length} registries, ` +
161163
`${metadata.errors.length} errors, ` +
162164
`downloaded ${this.crawler.formatBytes(metadata.totalBytes)}`);
163-
165+
this.stats.task('TxRegistry', 'Crawling Finished');
164166
} catch (error) {
165167
this.logger.error('Crawl failed:', error);
168+
this.stats.task('TxRegistry', 'Crawling Error: '+error.message);
166169
} finally {
167170
this.crawlInProgress = false;
168171
}

‎server.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,7 @@ async function buildRootPageContent() {
355355
historyJson: JSON.stringify(stats.history),
356356
startTime: stats.startTime
357357
});
358+
content += stats.taskDetails();
358359

359360
content += '</div>';
360361
return content;

‎stats.js‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
const { monitorEventLoopDelay } = require('perf_hooks');
2-
const {cache} = require("express/lib/application");
2+
const {Utilities} = require("./library/utilities");
33

44
class ServerStats {
55
started = false;
@@ -13,6 +13,7 @@ class ServerStats {
1313
startTime = Date.now();
1414
timer;
1515
cachingModules = [];
16+
taskMap = new Map();
1617

1718
constructor() {
1819
this.timer = setInterval(() => {
@@ -73,6 +74,35 @@ class ServerStats {
7374
this.requestTime = this.requestTime + tat;
7475
}
7576

77+
task(name, state) {
78+
let info = this.taskMap.get(name);
79+
if (!info) {
80+
info = {};
81+
this.taskMap.set(name, info);
82+
}
83+
info.date = Date.now();
84+
info.state = state;
85+
}
86+
87+
taskDetails() {
88+
if (this.taskMap.size == 0) {
89+
return "";
90+
}
91+
let html = '<table class="grid"><tr><th colspan="3">Background Tasks</th></tr>';
92+
html += "<tr><th>Task</th><th>Status</th><th>Last Seen</th></tr>";
93+
for (let m of this.taskMap.keys()) {
94+
html += "<tr><td>";
95+
html += Utilities.escapeHtml(m);
96+
html += "</td><td>";
97+
html += Utilities.escapeHtml(this.taskMap.get(m).state);
98+
html += "</td><td>";
99+
html += Utilities.formatDuration(this.taskMap.get(m).date, Date.now());
100+
html += "</td></tr>";
101+
}
102+
html += "</table>";
103+
return html;
104+
}
105+
76106
finishStats() {
77107
clearInterval(this.timer);
78108
}

‎tx/README.md‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,7 @@ The TX module provides FHIR terminology services for CodeSystem, ValueSet, and C
44

55
## Todo
66

7-
* More work on the HTML interface (external code systems, global functions, render capability statements)
8-
* add more tests for the code system providers - filters, extended lookup, designations and languages
9-
* more refactoring in validate.js and expand.js
10-
* full batch support
11-
* check vsac support
12-
* get tx tests running in pipelines
7+
* Improve batch support
138

149
## Overview
1510

‎tx/library.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,12 @@ class Library {
8484
}
8585
}
8686

87-
constructor(configFile, vsacCfg, log) {
87+
constructor(configFile, vsacCfg, log, stats) {
8888
this.configFile = configFile;
8989
this.vsacCfg = vsacCfg;
9090
this.log = log;
91+
this.stats = stats;
92+
9193
// Only synchronous initialization here
9294
this.codeSystemFactories = new Map();
9395
this.codeSystemProviders = [];
@@ -312,7 +314,7 @@ class Library {
312314
if (!this.vsacCfg || !this.vsacCfg.apiKey) {
313315
throw new Error("Unable to load VSAC provider unless vsacCfg is provided in the configuration");
314316
}
315-
let vsac = new VSACValueSetProvider(this.vsacCfg);
317+
let vsac = new VSACValueSetProvider(this.vsacCfg, this.stats);
316318
vsac.initialize();
317319
this.valueSetProviders.push(vsac);
318320
//const mem = process.memoryUsage();

0 commit comments

Comments
 (0)