From 1e3952f5788d8284375f5c24d6bfe9cfdbf34ae4 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Tue, 8 Apr 2025 18:37:13 +0300 Subject: [PATCH 1/8] feat: mdr enrollment --- config/custom-environment-variables.json | 17 +++ config/default.json | 8 ++ example/mdr.lua | 115 ++++++++++++++++++ helm/templates/configmap.yaml | 10 +- helm/values.yaml | 7 ++ src/cli/commands/append/appendManager.ts | 73 ++++++++--- .../commands/append/appendManagerFactory.ts | 8 +- src/cli/commands/create/createFactory.ts | 16 ++- src/commandRunner/osmCommandRunner.ts | 3 + src/common/constants.ts | 1 + src/common/interfaces.ts | 7 ++ src/common/util.ts | 10 ++ src/httpClient/mdrClient.ts | 55 +++++++++ 13 files changed, 306 insertions(+), 24 deletions(-) create mode 100644 example/mdr.lua create mode 100644 src/httpClient/mdrClient.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 9ad414b..9870735 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -54,6 +54,10 @@ "__name": "OSM2PGSQL_GENERATE_EXPIRE_OUTPUT", "__format": "boolean" }, + "extraAttributes": { + "__name": "OSM2PGSQL_EXTRA_ATTRIBUTES", + "__format": "boolean" + }, "schema": "OSM2PGSQL_SCHEMA", "middleSchema": "OSM2PGSQL_MIDDLE_SCHEMA", "logger": { @@ -102,6 +106,19 @@ }, "database": "PGBOSS_DATABASE" }, + "mdr": { + "enabled": { + "__name": "MDR_ENABLED", + "__format": "boolean" + }, + "client": { + "url": "MDR_CLIENT_URL", + "timeout": { + "__name": "MDR_CLIENT_TIMEOUT", + "__format": "number" + } + } + }, "arstotzka": { "enabled": { "__name": "ARSTOTZKA_ENABLED", diff --git a/config/default.json b/config/default.json index b3bccdc..e9bc63c 100644 --- a/config/default.json +++ b/config/default.json @@ -25,6 +25,7 @@ "processes": 2, "output": "flex", "generateExpireOutput": true, + "extraAttributes": false, "logger": { "level": "info", "progress": true, @@ -36,6 +37,13 @@ "verbose": false, "progress": false }, + "mdr": { + "enabled": false, + "client": { + "url": "http://mdr.com", + "timeout": 10000 + } + }, "arstotzka": { "enabled": false, "serviceId": "serviceId", diff --git a/example/mdr.lua b/example/mdr.lua new file mode 100644 index 0000000..90d13fd --- /dev/null +++ b/example/mdr.lua @@ -0,0 +1,115 @@ +local SRID = 4326 +local GFID_TAG = 'gfid_tag' +local HISTORY_ID_TAG = 'history_id_tag' +local ALTITUDE_TAG = 'altitude' +local DEFAULT_ALTITUDE = 0 + +function object_to_geom(object) + if object.is_closed then return object:as_polygon() else return object:as_linestring() end +end + +local tables = {} + +tables.dependent_nodes = osm2pgsql.define_table({ + name = 'dependent_nodes', + ids = { type = 'node', id_column = 'osm_id' }, + columns = { + { column = 'geom', type = 'point', projection = SRID, not_null = true }, + { column = 'altitude', type = 'real' }, + } +}) + +tables.nodes = osm2pgsql.define_table({ + name = 'nodes', + ids = { type = 'node', id_column = 'osm_id' }, + columns = { + { column = 'gfid', type = 'text' }, + { column = 'history_id', type = 'text' }, + { column = 'geom', type = 'point', projection = SRID, not_null = true }, + { column = 'altitude', type = 'real' }, + { column = 'tags', type = 'jsonb' } + } +}) + +tables.ways = osm2pgsql.define_table({ + name = 'ways', + ids = { type = 'way', id_column = 'osm_id' }, + columns = { + { column = 'gfid', type = 'text' }, + { column = 'history_id', type = 'text' }, + { column = 'geom', type = 'geometry', projection = SRID, not_null = true }, + { column = 'nodes', type = 'text', sql_type = 'bigint[]' }, + { column = 'tags', type = 'jsonb' } + } +}) + +tables.expired = osm2pgsql.define_table{ + name = "expired", + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'serial_id', sql_type = 'serial', create_only = true }, + { column = 'gfid', type = 'text' }, + { column = 'history_id', type = 'text' } + } +} + +function insert_safely(table_key, insert_object) + if not tables or not tables[table_key] then + error("Table '" .. tostring(table_key) .. "' does not exist.") + end + + local inserted, message, column, object = tables[table_key]:insert(insert_object) + + if not inserted then + print("Insert failed: ", message) + print("gfid ", insert_object.gfid) + print("column ", column) + print("object ", object) + end + + return true +end + +function osm2pgsql.process_node(object) + local gfid = object:grab_tag(GFID_TAG) + local history_id = object:grab_tag(HISTORY_ID_TAG) + local altitude = object:grab_tag(ALTITUDE_TAG) + + if gfid then + insert_safely("nodes", { + gfid = gfid, + history_id = history_id, + geom = object:as_point(), + altitude = altitude, + tags = object.tags + }) + + insert_safely("expired", { + gfid = gfid, + history_id = history_id + }) + elseif altitude then + insert_safely("dependent_nodes", { + geom = object:as_point(), + altitude = altitude + }) + end +end + +function osm2pgsql.process_way(object) + local gfid = object:grab_tag(GFID_TAG) + local history_id = object:grab_tag(HISTORY_ID_TAG) + + insert_safely("ways", { + gfid = gfid, + history_id = history_id, + geom = object_to_geom(object), + nodes = '{' .. table.concat(object.nodes, ',') .. '}', + tags = object.tags + }) + + insert_safely("expired", { + gfid = gfid, + history_id = history_id + }) +end diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index a2afb49..564d578 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -29,7 +29,8 @@ data: TELEMETRY_METRICS_URL: {{ $metricsUrl }} {{ end }} HTTP_CLIENT_TIMEOUT: {{ .Values.env.httpClient.timeout | quote }} - OSM2PGSQL_SLIM: {{ .Values.env.osm2pgsql.slim | quote}} + OSM2PGSQL_SLIM: {{ .Values.env.osm2pgsql.slim | quote }} + OSM2PGSQL_EXTRA_ATTRIBUTES: {{ .Values.env.osm2pgsql.extraAttributes | quote }} OSM2PGSQL_CACHE: {{ .Values.env.osm2pgsql.cache | quote }} OSM2PGSQL_PROCESSES: {{ .Values.env.osm2pgsql.processes | quote }} OSM2PGSQL_OUTPUT: {{ .Values.env.osm2pgsql.output | quote }} @@ -42,6 +43,13 @@ data: OSM2PGSQL_MIDDLE_SCHEMA: {{ .Values.env.osm2pgsql.middleSchema | quote }} OSMIUM_VERBOSE: {{ .Values.env.osmium.verbose | quote }} OSMIUM_PROGRESS: {{ .Values.env.osmium.progress | quote }} + {{- if .Values.mdr.enabled }} + MDR_ENABLED: "true" + MDR_CLIENT_URL: {{ .Values.mdr.client.url | quote }} + MDR_CLIENT_TIMEOUT: {{ .Values.mdr.client.timeout | quote }} + {{- else }} + MDR_ENABLED: "false" + {{- end }} {{- with .Values.pgboss }} PGBOSS_HOST: {{ .host | quote }} PGBOSS_PORT: {{ .port | default 5432 | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 615424b..258b1a0 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -66,6 +66,7 @@ env: timeout: 1000 osm2pgsql: slim: true + extraAttributes: false cache: 2500 processes: 2 output: flex @@ -137,6 +138,12 @@ cli: dumpSource: 'https://remote-url.com' s3LuaScriptKey: script.lua +mdr: + enabled: false + client: + url: 'https://mdr.com' + timeout: 10000 + arstotzka: enabled: false serviceId: id diff --git a/src/cli/commands/append/appendManager.ts b/src/cli/commands/append/appendManager.ts index ad506e9..b28fd93 100644 --- a/src/cli/commands/append/appendManager.ts +++ b/src/cli/commands/append/appendManager.ts @@ -10,7 +10,14 @@ import { ActionStatus } from '@map-colonies/arstotzka-common'; import client from 'prom-client'; import { IConfig, RemoteResource } from '../../../common/interfaces'; import { DATA_DIR, SERVICES, DIFF_FILE_EXTENTION, EXPIRE_LIST, METRICS_BUCKETS } from '../../../common/constants'; -import { streamToUniqueLines, getDiffDirPathComponents, streamToFs, valuesToRange } from '../../../common/util'; +import { + streamToUniqueLines, + getDiffDirPathComponents, + streamToFs, + valuesToRange, + shouldEnrollMdr, + getMdrEnrollmentRange, +} from '../../../common/util'; import { ReplicationClient } from '../../../httpClient/replicationClient'; import { AppendEntity } from '../../../validation/schemas'; import { S3ClientWrapper } from '../../../s3Client/s3Client'; @@ -19,6 +26,7 @@ import { OsmCommandRunner } from '../../../commandRunner/osmCommandRunner'; import { QueueProvider } from '../../../queue/queueProvider'; import { RequestAlreadyInQueueError } from '../../../common/errors'; import { RemoteResourceManager } from '../../../remoteResource/remoteResourceManager'; +import { EnrollmentStatus, IMdrClient } from '../../../httpClient/mdrClient'; import { terminateChildren } from '../../../commandRunner/spawner'; import { QueueSettings, TileRequestQueuePayload } from './interfaces'; import { StateTracker } from './stateTracker'; @@ -49,7 +57,8 @@ export class AppendManager { private readonly remoteResourceManager: RemoteResourceManager, private readonly queueProvider?: QueueProvider, @inject(SERVICES.METRICS_REGISTRY) registry?: client.Registry, - @inject(METRICS_BUCKETS) metricsBuckets?: number[] + @inject(METRICS_BUCKETS) metricsBuckets?: number[], + private readonly mdrClient?: IMdrClient ) { this.shouldGenerateExpireOutput = config.get('osm2pgsql.generateExpireOutput'); if (configStore.has('queue')) { @@ -144,15 +153,7 @@ export class AppendManager { await mediator?.removeLock(); try { - await this.appendNextState(replicationUrl); - - await this.stateTracker.updateRemoteTimestamp(); - - if (this.shouldGenerateExpireOutput) { - await this.uploadExpired(); - } - - await this.stateTracker.updateRemoteState(); + await this.appendProcedure(replicationUrl); } catch (error) { terminateChildren(); await mediator?.updateAction({ status: ActionStatus.FAILED, metadata: { error } }); @@ -204,15 +205,7 @@ export class AppendManager { await mediator?.removeLock(); try { - await this.appendNextState(replicationUrl); - - await this.stateTracker.updateRemoteTimestamp(); - - if (this.shouldGenerateExpireOutput) { - await this.uploadExpired(); - } - - await this.stateTracker.updateRemoteState(); + await this.appendProcedure(replicationUrl); this.appendsCounter?.inc({ status: 'completed' }); } catch (error) { @@ -234,6 +227,46 @@ export class AppendManager { } } + private async appendProcedure(replicationUrl: string): Promise { + // if mdr is enabled get pre append status + const preMdrStatus = await this.mdrClient?.getStatus(); + + // actual osm2pg append + await this.appendNextState(replicationUrl); + + // update remote's timestamp + await this.stateTracker.updateRemoteTimestamp(); + + // if mdr is enabled get post append status + const postMdrStatus = await this.mdrClient?.getStatus(); + + // if needed upload expired + if (this.shouldGenerateExpireOutput) { + await this.uploadExpired(); + } + + // if needed enroll on mdr + if (shouldEnrollMdr(preMdrStatus, postMdrStatus)) { + await this.mdrClient?.postEnrollment({ + state: this.stateTracker.nextState, + isFull: false, + ...getMdrEnrollmentRange(preMdrStatus as EnrollmentStatus, postMdrStatus as EnrollmentStatus), + }); + } + + // update remote's state + await this.stateTracker.updateRemoteState(); + + this.logger.info({ + msg: 'append procedure completed', + state: this.stateTracker.nextState, + enabledGenerateExpireOutput: this.shouldGenerateExpireOutput, + enabledMdr: this.mdrClient !== undefined, + preStatus: preMdrStatus, + postStatus: postMdrStatus, + }); + } + private async appendNextState(replicationUrl: string): Promise { this.logger.info({ msg: 'attempting to append state on entities', diff --git a/src/cli/commands/append/appendManagerFactory.ts b/src/cli/commands/append/appendManagerFactory.ts index b231cd6..7226abe 100644 --- a/src/cli/commands/append/appendManagerFactory.ts +++ b/src/cli/commands/append/appendManagerFactory.ts @@ -3,13 +3,14 @@ import { Registry } from 'prom-client'; import { FactoryFunction } from 'tsyringe'; import { OsmCommandRunner } from '../../../commandRunner/osmCommandRunner'; import { SERVICES, METRICS_BUCKETS } from '../../../common/constants'; -import { IConfig } from '../../../common/interfaces'; +import { IConfig, MdrConfig } from '../../../common/interfaces'; import { RemoteResourceManager } from '../../../remoteResource/remoteResourceManager'; import { S3RemoteResourceProvider } from '../../../remoteResource/s3ResourceProvider'; import { ReplicationClient } from '../../../httpClient/replicationClient'; import { QUEUE_PROVIDER_SYMBOL } from '../../../queue/constants'; import { QueueProvider } from '../../../queue/queueProvider'; import { S3ClientWrapper } from '../../../s3Client/s3Client'; +import { MdrClient } from '../../../httpClient/mdrClient'; import { AppendManager } from './appendManager'; import { StateTracker } from './stateTracker'; @@ -27,6 +28,8 @@ export const appendManagerFactory: FactoryFunction = (dependencyC : undefined; const registry = dependencyContainer.resolve(SERVICES.METRICS_REGISTRY); const metricsBuckets = dependencyContainer.resolve(METRICS_BUCKETS); + const mdrConfig = config.get('mdr'); + const mdrClient = mdrConfig.enabled ? new MdrClient({ ...mdrConfig.client, logger: logger.child({ component: 'mdr' }) }) : undefined; return new AppendManager( logger, @@ -39,6 +42,7 @@ export const appendManagerFactory: FactoryFunction = (dependencyC remoteResourceManager, queueProv, registry, - metricsBuckets + metricsBuckets, + mdrClient ); }; diff --git a/src/cli/commands/create/createFactory.ts b/src/cli/commands/create/createFactory.ts index 7eb209e..177767a 100644 --- a/src/cli/commands/create/createFactory.ts +++ b/src/cli/commands/create/createFactory.ts @@ -3,12 +3,13 @@ import { Logger } from '@map-colonies/js-logger'; import { FactoryFunction } from 'tsyringe'; import { StatefulMediator } from '@map-colonies/arstotzka-mediator'; import { ActionStatus } from '@map-colonies/arstotzka-common'; -import { ArstotzkaConfig } from '../../../common/interfaces'; +import { ArstotzkaConfig, IConfig, MdrConfig } from '../../../common/interfaces'; import { GlobalArguments } from '../../cliBuilderFactory'; import { ExitCodes, EXIT_CODE, SERVICES } from '../../../common/constants'; import { ErrorWithExitCode } from '../../../common/errors'; import { dumpSourceCheck } from '../../checks'; import { ValidationResponse } from '../../../validation/validator'; +import { IMdrClient, MdrClient } from '../../../httpClient/mdrClient'; import { CreateManager } from './createManager'; import { command, describe, CREATE_MANAGER_FACTORY, DumpSourceType } from './constants'; @@ -22,6 +23,8 @@ export interface CreateArguments extends GlobalArguments { export const createCommandFactory: FactoryFunction> = (dependencyContainer) => { const logger = dependencyContainer.resolve(SERVICES.LOGGER); + const config = dependencyContainer.resolve(SERVICES.CONFIG); + const builder = (args: Argv): Argv => { args .option('dumpSourceType', { @@ -66,6 +69,12 @@ export const createCommandFactory: FactoryFunction('mdr'); + let mdrClient: IMdrClient | undefined; + if (mdrConfig.enabled) { + mdrClient = new MdrClient({ logger: logger.child({ component: 'mdr' }), ...mdrConfig.client }); + } + const manager = dependencyContainer.resolve(CREATE_MANAGER_FACTORY); try { @@ -82,6 +91,11 @@ export const createCommandFactory: FactoryFunction { return arr.sort((a, b) => (sort === 'desc' ? b.localeCompare(a) : a.localeCompare(b))); }; + +export const shouldEnrollMdr = (pre: EnrollmentStatus | undefined, post: EnrollmentStatus | undefined): boolean => + pre !== undefined && post !== undefined && (pre.count !== post.count || pre.latest !== post.latest); + +export const getMdrEnrollmentRange = (pre: EnrollmentStatus, post: EnrollmentStatus): { from: number; to: number } => { + const from = pre.latest !== undefined ? pre.latest + 1 : 0; + const to = post.latest as number; + return { from, to }; +}; diff --git a/src/httpClient/mdrClient.ts b/src/httpClient/mdrClient.ts new file mode 100644 index 0000000..d45bd74 --- /dev/null +++ b/src/httpClient/mdrClient.ts @@ -0,0 +1,55 @@ +import { injectable } from 'tsyringe'; +import axios, { AxiosInstance } from 'axios'; +import { ILogger } from '../common/interfaces'; + +interface EnrollmentRequestBody { + state: number; + isFull: boolean; + from: number; + to: number; + metadata?: Record; +} + +export interface EnrollmentStatus { + count: number; + latest?: number; +} + +export interface MdrClientOptions { + url: string; + timeout: number; + logger?: ILogger; +} + +export interface IMdrClient { + getStatus: () => Promise; + postEnrollment: (enrollment: EnrollmentRequestBody) => Promise; +} + +@injectable() +export class MdrClient implements IMdrClient { + private readonly httpClient: AxiosInstance; + private readonly logger: ILogger | undefined; + + public constructor(options: MdrClientOptions) { + const { logger, ...clientOptions } = options; + this.httpClient = axios.create({ baseURL: clientOptions.url, timeout: clientOptions.timeout }); + this.logger = logger; + + this.logger?.info({ msg: 'initialized mdr client', clientOptions }); + } + + public async getStatus(): Promise { + this.logger?.info({ msg: 'getting current enrollment status' }); + + const { data } = await this.httpClient.get('/enrollment/status'); + + return data; + } + + public async postEnrollment(enrollment: EnrollmentRequestBody): Promise { + this.logger?.info({ msg: 'posting enrollment request', enrollment }); + + await this.httpClient.post('/enrollment', enrollment); + } +} From 95e2ef5046c49d958ce54bce69248a9c02191260 Mon Sep 17 00:00:00 2001 From: melancholiai Date: Wed, 9 Apr 2025 18:29:03 +0300 Subject: [PATCH 2/8] refactor: minor changes and missing values --- helm/templates/command-options-configmap.yaml | 1 + helm/values.yaml | 1 + src/cli/commands/append/appendManager.ts | 18 +++++++++--------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/helm/templates/command-options-configmap.yaml b/helm/templates/command-options-configmap.yaml index 18aa654..242ed85 100644 --- a/helm/templates/command-options-configmap.yaml +++ b/helm/templates/command-options-configmap.yaml @@ -30,5 +30,6 @@ data: DUMP_SOURCE_TYPE: {{ .Values.cli.create.dumpSourceType | quote }} DUMP_SOURCE: {{ .Values.cli.create.dumpSource | quote }} S3_LUA_SCRIPT_KEY: {{ .Values.cli.create.s3LuaScriptKey | quote }} + STATE_OVERRIDE: {{ .Values.cli.create.stateOverride | quote }} {{- end }} {{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 258b1a0..d90893a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -137,6 +137,7 @@ cli: dumpSourceType: remote-url # local-file, remote-url or dump-server dumpSource: 'https://remote-url.com' s3LuaScriptKey: script.lua + # stateOverride: mdr: enabled: false diff --git a/src/cli/commands/append/appendManager.ts b/src/cli/commands/append/appendManager.ts index b28fd93..80f6bf5 100644 --- a/src/cli/commands/append/appendManager.ts +++ b/src/cli/commands/append/appendManager.ts @@ -228,24 +228,19 @@ export class AppendManager { } private async appendProcedure(replicationUrl: string): Promise { - // if mdr is enabled get pre append status + // get pre append status if enabled const preMdrStatus = await this.mdrClient?.getStatus(); - // actual osm2pg append + // execute osm2pg append await this.appendNextState(replicationUrl); // update remote's timestamp await this.stateTracker.updateRemoteTimestamp(); - // if mdr is enabled get post append status + // get post append mdr status if enabled const postMdrStatus = await this.mdrClient?.getStatus(); - // if needed upload expired - if (this.shouldGenerateExpireOutput) { - await this.uploadExpired(); - } - - // if needed enroll on mdr + // enroll on mdr if needed if (shouldEnrollMdr(preMdrStatus, postMdrStatus)) { await this.mdrClient?.postEnrollment({ state: this.stateTracker.nextState, @@ -254,6 +249,11 @@ export class AppendManager { }); } + // upload expired if needed + if (this.shouldGenerateExpireOutput) { + await this.uploadExpired(); + } + // update remote's state await this.stateTracker.updateRemoteState(); From 9b382bb235a20592656b91cf5f57c35a80cdcfcd Mon Sep 17 00:00:00 2001 From: melancholiai Date: Thu, 24 Apr 2025 16:00:25 +0300 Subject: [PATCH 3/8] refactor(deps): updated deps to be up to date --- .gitattributes | 1 + .github/ISSUE_TEMPLATE/bug_report.md | 3 +- .github/ISSUE_TEMPLATE/feature_request.md | 1 - .github/PULL_REQUEST_TEMPLATE.md | 21 +- .github/dependabot.yaml | 53 + .github/workflows/build-and-push.yaml | 31 + .github/workflows/pull_request.yaml | 55 +- .github/workflows/release-on-tag-push.yaml | 20 - .github/workflows/release-please.yaml | 21 + .husky/commit-msg | 1 + .husky/install.mjs | 6 + .husky/pre-commit | 1 + .nvmrc | 2 +- .prettierignore | 1 + .release-please-manifest.json | 1 + .swcrc | 8 + .versionrc.json | 16 - .vscode/extensions.json | 20 + .vscode/launch.json | 6 +- .vscode/settings.json | 6 + Dockerfile | 22 +- README.md | 10 +- azure-pipelines.yml | 45 - catalog-info.yaml | 2 +- commitlint.config.js | 2 +- config/custom-environment-variables.json | 4 +- config/default.json | 7 +- eslint.config.mjs | 5 + helm/README.md | 3 +- helm/config/mdr-config.json | 6 + helm/templates/_helpers.tpl | 62 +- helm/templates/arstotzka-configmap.yaml | 4 +- helm/templates/command-options-configmap.yaml | 4 +- helm/templates/configmap.yaml | 6 +- helm/templates/cronjob.yaml | 103 - helm/templates/deployment.yaml | 126 +- helm/templates/job.yaml | 106 +- helm/templates/schema-configmap.yaml | 3 +- helm/templates/secret.yaml | 2 +- helm/values.yaml | 78 +- package-lock.json | 40769 +++++----------- package.json | 100 +- release-please-config.json | 21 + src/cli/commands/append/appendFactory.ts | 4 +- src/cli/commands/append/appendManager.ts | 41 +- .../commands/append/appendManagerFactory.ts | 5 +- src/cli/commands/append/expireTilesFilters.ts | 7 +- src/cli/commands/append/expireTilesParser.ts | 21 +- src/cli/commands/append/interfaces.ts | 2 +- src/cli/commands/append/stateTracker.ts | 2 +- src/cli/commands/create/createFactory.ts | 2 +- src/cli/commands/create/createManager.ts | 12 +- src/cli/middlewares/index.ts | 4 +- src/commandRunner/osmCommandRunner.ts | 9 +- src/common/config.ts | 38 + src/common/configStore.ts | 6 +- src/common/constants.ts | 8 +- src/common/errors.ts | 5 +- src/common/interfaces.ts | 6 +- src/common/liveness.ts | 2 +- src/common/tracing.ts | 35 +- src/common/util.ts | 11 +- src/containerConfig.ts | 40 +- src/httpClient/baseClient.ts | 4 +- src/httpClient/dumpClient.ts | 13 +- src/httpClient/replicationClient.ts | 9 +- src/index.ts | 12 +- src/instrumentation.mts | 16 + src/queue/pgBossFactory.ts | 6 +- src/queue/pgBossQueueProvider.ts | 14 +- src/remoteResource/remoteResourceManager.ts | 12 +- src/s3Client/s3Client.ts | 4 +- src/validation/validator.ts | 21 +- start.sh | 2 +- tests/configurations/jest.setup.ts | 1 + tests/configurations/unit/jest.config.js | 15 +- .../expireTilesParser.spec.ts | 76 +- tests/unit/expireTilesParser/helper.ts | 10 +- tsconfig.build.json | 7 +- tsconfig.json | 22 +- tsconfig.lint.json | 8 - tsconfig.test.json | 9 - 82 files changed, 12759 insertions(+), 29526 deletions(-) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/build-and-push.yaml delete mode 100644 .github/workflows/release-on-tag-push.yaml create mode 100644 .github/workflows/release-please.yaml create mode 100644 .husky/commit-msg create mode 100644 .husky/install.mjs create mode 100644 .husky/pre-commit create mode 100644 .release-please-manifest.json create mode 100644 .swcrc delete mode 100644 .versionrc.json create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json delete mode 100644 azure-pipelines.yml create mode 100644 eslint.config.mjs create mode 100644 helm/config/mdr-config.json delete mode 100644 helm/templates/cronjob.yaml create mode 100644 release-please-config.json create mode 100644 src/common/config.ts create mode 100644 src/instrumentation.mts create mode 100644 tests/configurations/jest.setup.ts delete mode 100644 tsconfig.lint.json delete mode 100644 tsconfig.test.json diff --git a/.gitattributes b/.gitattributes index 314b2ee..3b5aed2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ #prevent git from chaning line endings of sh file to /r/n on windows systems *.sh text eol=lf +*.* text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 03d32eb..ac41781 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,7 +4,6 @@ about: Create a report to help us improve title: '' labels: 'bug' assignees: '' - --- **Describe the bug** @@ -12,6 +11,7 @@ A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' @@ -23,6 +23,5 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. - **Additional context** Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 36014cd..104f391 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,7 +4,6 @@ about: Suggest an idea for this project title: '' labels: 'enhancement' assignees: '' - --- **Is your feature request related to a problem? Please describe.** diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9435ba3..c26ec58 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,20 +2,21 @@ Make sure you've read the contributing guidelines (CONTRIBUTING.md) --> -| Question | Answer | -| ---------------- | -------------------------------------------------------------------------- | -| Bug fix | ✔/✖ | -| New feature | ✔/✖ | -| Breaking change | ✔/✖ | -| Deprecations | ✔/✖ | -| Documentation | ✔/✖ | -| Tests added | ✔/✖ | -| Chore | ✔/✖ | +| Question | Answer | +| --------------- | ------ | +| Bug fix | ✔/✖ | +| New feature | ✔/✖ | +| Breaking change | ✔/✖ | +| Deprecations | ✔/✖ | +| Documentation | ✔/✖ | +| Tests added | ✔/✖ | +| Chore | ✔/✖ | Related issues: #XXX , #XXX ... Closes #XXX ... -Further information: +Further information: +