diff --git a/package-lock.json b/package-lock.json index 6a8f834..4c1f537 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "zibri", - "version": "2.1.1", + "version": "2.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "zibri", - "version": "2.1.1", + "version": "2.1.2", "license": "MIT", "dependencies": { "@fastify/busboy": "^3.2.0", @@ -15706,9 +15706,9 @@ } }, "node_modules/systeminformation": { - "version": "5.27.13", - "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.27.13.tgz", - "integrity": "sha512-geeE/7eNDoOhdc9j+qCsLlwbcyh0HnqhOZzmfNK4WBioWGUZbhwYrg+YZsZ3UJh4tmybQsnDuqzr3UoumMifew==", + "version": "5.27.17", + "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.27.17.tgz", + "integrity": "sha512-aWH+WoKHOGV5qBI9IPzFSwmLXrUSukF8aKXhWDZU990kKs65b+LSD69/1P0PKDweu9o8oCMW9QkJGG9R8FLi5A==", "license": "MIT", "os": [ "darwin", diff --git a/package.json b/package.json index 5c88e75..fa5096d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "zibri", - "version": "2.1.1", + "version": "2.1.2", "main": "./dist/index.js", "module": "./dist/index.mjs", "repository": { diff --git a/sandbox/src/data-sources/db/db.data-source.ts b/sandbox/src/data-sources/db/db.data-source.ts index 7e485b0..aea6e37 100644 --- a/sandbox/src/data-sources/db/db.data-source.ts +++ b/sandbox/src/data-sources/db/db.data-source.ts @@ -1,4 +1,4 @@ -import { PostgresDataSource, PostgresOptions, BaseEntity, DataSource, Newable, MigrationEntity, JwtRefreshToken, JwtCredentials, PasswordResetToken, MailingList, MailingListSubscriber, MailingListSubscriptionConfirmationToken, Log, Change, ChangeSet, Invoice, NumberInvoices, Entity, OmitClass, OtpCredentials, BackupResourceEntity, BackupEntity } from 'zibri'; +import { PostgresDataSource, PostgresOptions, BaseEntity, DataSource, Newable, MigrationEntity, JwtRefreshToken, JwtCredentials, PasswordResetToken, MailingList, MailingListSubscriber, MailingListSubscriptionConfirmationToken, Log, Change, ChangeSet, Entity, OmitClass, OtpCredentials, BackupResourceEntity, BackupEntity, Invoice, NumberInvoices } from 'zibri'; import { Company, Test, User } from '../../models'; diff --git a/sandbox/src/index.ts b/sandbox/src/index.ts index b50da84..9444ada 100644 --- a/sandbox/src/index.ts +++ b/sandbox/src/index.ts @@ -2,10 +2,10 @@ import H from 'handlebars/runtime'; import { inject, isVersion, JwtAuthController, LoggerInterface, ZIBRI_DI_TOKENS, ZibriApplication, ZibriInvoicingPlugin } from 'zibri'; import { CronController, FileController, MetricsController, TemplateController, TestController, TestCrudController, TestWebsocketController } from './controllers'; -import { DbDataSource } from './data-sources'; -import { version } from '../package.json'; import { createDefaultData } from './create-default-data.function'; import { StatusCronJob } from './cron'; +import { DbDataSource } from './data-sources'; +import { version } from '../package.json'; import { providers } from './providers'; export let logger: LoggerInterface; diff --git a/src/backup/backup-entity.model.ts b/src/backup/backup-entity.model.ts index 1892e21..cf069cf 100644 --- a/src/backup/backup-entity.model.ts +++ b/src/backup/backup-entity.model.ts @@ -1,9 +1,9 @@ import { inject, ZIBRI_DI_TOKENS } from '../di'; import { Entity, Property } from '../entity'; +import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model'; import { BaseEntity } from '../entity/base-entity.model'; import { FormatDateFn } from '../localization'; import { OmitStrict } from '../types'; -import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model'; /** * The entity of a single backup. diff --git a/src/backup/backup-resource-entity.model.ts b/src/backup/backup-resource-entity.model.ts index caea82c..044e3a3 100644 --- a/src/backup/backup-resource-entity.model.ts +++ b/src/backup/backup-resource-entity.model.ts @@ -1,7 +1,7 @@ import { Entity, Property } from '../entity'; +import { BackupEntity } from './backup-entity.model'; import { BaseEntity } from '../entity/base-entity.model'; import { OmitStrict } from '../types'; -import { BackupEntity } from './backup-entity.model'; /** * A single resource entity. diff --git a/src/backup/backup-service.interface.ts b/src/backup/backup-service.interface.ts index fa92c16..6e80a74 100644 --- a/src/backup/backup-service.interface.ts +++ b/src/backup/backup-service.interface.ts @@ -29,6 +29,10 @@ export interface BackupServiceInterface { * Initializes the service. */ init: () => void | Promise, + /** + * Synchronizes backup entities with the data from all transports. + */ + syncBackupEntities: () => Promise, /** * Starts a new backup based on the given configuration. */ diff --git a/src/backup/backup-service.test.ts b/src/backup/backup-service.test.ts index bd5eee0..9c4675e 100644 --- a/src/backup/backup-service.test.ts +++ b/src/backup/backup-service.test.ts @@ -89,6 +89,10 @@ describe('Create and restore postgres backup', () => { expect((await itemRepository.findAll()).length).toEqual(1); await itemRepository.deleteAll({}); expect((await itemRepository.findAll()).length).toEqual(0); + await backupRepository.deleteAll({}); + expect((await backupRepository.findAll()).length).toEqual(0); + await backupService.syncBackupEntities(); + expect((await backupRepository.findAll()).length).toEqual(1); const backup: BackupEntity = (await backupRepository.findAll({ relations: ['resources'] }))[0]; await backupService.restore(backup); expect((await itemRepository.findAll()).length).toEqual(1); diff --git a/src/backup/backup.service.ts b/src/backup/backup.service.ts index e4483d8..6bb842a 100644 --- a/src/backup/backup.service.ts +++ b/src/backup/backup.service.ts @@ -7,7 +7,7 @@ import { inject, repositoryTokenFor, ZIBRI_DI_TOKENS } from '../di'; import { GlobalRegistry } from '../global'; import { LoggerInterface } from '../logging'; import { Newable } from '../types'; -import { MetadataUtilities, validateEntitiesRegistered } from '../utilities'; +import { chunkedPromiseAll, MetadataUtilities, validateEntitiesRegistered } from '../utilities'; import { BackupEntity, BackupEntityCreateData } from './backup-entity.model'; import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model'; import { BackupResourceInterface } from './backup-resource.interface'; @@ -58,6 +58,62 @@ export class BackupService implements BackupServiceInterface { } await this.logger.info(` - ${resourceClass.name}`); } + + await this.syncBackupEntities(); + } + + // eslint-disable-next-line jsdoc/require-jsdoc + async syncBackupEntities(): Promise { + if (!this.backupResources.length) { + return; + } + const transports: BackupTransportInterface[] = this.backupResources + .map(r => MetadataUtilities.getBackupResourceMetadata(r)?.transports ?? []) + .flat(); + if (!transports.length) { + await this.logger.warn('Could not find any transports to sync backup entities from'); + return; + } + const existingEntities: BackupEntity[] = await this.backupRepository.findAll(); + const resolvedEntities: BackupEntity[] = (await Promise.all(transports.map(t => t.resolveBackups(existingEntities)))).flat(); + const entitiesToSync: BackupEntity[] = resolvedEntities.filter(entity => !existingEntities.map(e => e.id).includes(entity.id)); + + const groupedEntities: Record = this.groupEntitiesById(entitiesToSync); + const mergedEntities: BackupEntity[] = this.mergeEntities(groupedEntities); + await chunkedPromiseAll(mergedEntities.map(e => this.backupRepository.create(e, { allowId: true }))); + } + + private mergeEntities(groupedEntities: Record): BackupEntity[] { + const res: BackupEntity[] = []; + for (const key in groupedEntities) { + const entities: BackupEntity[] = groupedEntities[key]; + res.push({ + createdAt: entities[0].createdAt, + id: entities[0].id, + name: entities[0].name, + resources: entities.reduce( + (prev, curr) => [ + ...prev, + ...curr.resources.filter(r => !prev.find(pr => pr.id === r.id)) + ], + [] + ) + }); + } + return res; + } + + private groupEntitiesById(entities: BackupEntity[]): Record { + const res: Record = {}; + for (const entity of entities) { + if (res[entity.id] == undefined) { + res[entity.id] = [entity]; + } + else { + res[entity.id].push(entity); + } + } + return res; } // eslint-disable-next-line jsdoc/require-jsdoc diff --git a/src/backup/decorators/backup-resource.decorator.ts b/src/backup/decorators/backup-resource.decorator.ts index 7b3b2ca..bd445ac 100644 --- a/src/backup/decorators/backup-resource.decorator.ts +++ b/src/backup/decorators/backup-resource.decorator.ts @@ -1,10 +1,10 @@ +import { BackupResourceMetadata, BackupResourceMetadataInput } from './backup-resource-metadata.model'; import { DiToken } from '../../di'; import { GlobalRegistry } from '../../global'; import { Newable } from '../../types'; import { MetadataUtilities } from '../../utilities'; import { BackupResourceInterface } from '../backup-resource.interface'; -import { BackupResourceMetadata, BackupResourceMetadataInput } from './backup-resource-metadata.model'; // eslint-disable-next-line jsdoc/require-returns /** diff --git a/src/backup/transports/backup-transport.interface.ts b/src/backup/transports/backup-transport.interface.ts index df1fc63..bdda6c7 100644 --- a/src/backup/transports/backup-transport.interface.ts +++ b/src/backup/transports/backup-transport.interface.ts @@ -11,6 +11,10 @@ export interface BackupTransportInterface { * The name of the transport. Is used to resolve the transport when restoring backups, so this should not change. */ readonly name: string, + /** + * Resolves all backups that have data stored over this transport. + */ + resolveBackups: (existingEntities: BackupEntity[]) => BackupEntity[] | Promise, /** * Stores the given data somewhere. */ diff --git a/src/backup/transports/fs.backup-transport.ts b/src/backup/transports/fs.backup-transport.ts index 88edccb..19671b6 100644 --- a/src/backup/transports/fs.backup-transport.ts +++ b/src/backup/transports/fs.backup-transport.ts @@ -1,23 +1,59 @@ -import { createReadStream, createWriteStream } from 'fs'; -import { mkdir, rm } from 'fs/promises'; +import { createReadStream, createWriteStream, Dirent } from 'fs'; +import { mkdir, readdir, readFile, rm, writeFile } from 'fs/promises'; import path from 'path'; import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; import { BackupTransportInterface } from './backup-transport.interface'; +import { inject, ZIBRI_DI_TOKENS } from '../../di'; +import { LoggerInterface } from '../../logging'; +import { pathExists } from '../../utilities'; import { BackupEntity } from '../backup-entity.model'; import { BackupResourceEntity } from '../backup-resource-entity.model'; +const METADATA_FILENAME: string = 'metadata.json'; + /** * A transport that saves the data to the local file system. */ export class FsBackupTransport implements BackupTransportInterface { + + private get logger(): LoggerInterface { + return inject(ZIBRI_DI_TOKENS.LOGGER); + } + constructor(readonly name: string, protected readonly backupBasePath: string) {} + // eslint-disable-next-line jsdoc/require-jsdoc + async resolveBackups(existingEntities: BackupEntity[]): Promise { + if (!await pathExists(this.backupBasePath)) { + return []; + } + + const nodes: Dirent[] = await readdir(this.backupBasePath, { withFileTypes: true }); + const res: BackupEntity[] = (await Promise.all(nodes.map(async node => { + if (!node.isDirectory()) { + return; + } + if (existingEntities.map(e => e.name).includes(node.name)) { + return; + } + const p: string = path.join(node.parentPath, node.name, METADATA_FILENAME); + if (!await pathExists(p)) { + await this.logger.warn(`Could not find the metadata file needed to resolve a backup: "${p}"`); + return; + } + const json: string = await readFile(p, 'utf8'); + return JSON.parse(json) as BackupEntity; + }))).filter(b => b != undefined); + return res; + } + // eslint-disable-next-line jsdoc/require-jsdoc async storeData(data: Readable, backup: BackupEntity, resource: BackupResourceEntity): Promise { - const p: string = this.getPath(backup, resource); - await mkdir(path.join(this.backupBasePath, backup.name), { recursive: true }); + const p: string = this.getResourcePath(backup, resource); + await mkdir(this.getBackupPath(backup), { recursive: true }); + await writeFile(this.getBackupMetadataPath(backup), JSON.stringify(backup), 'utf8'); await pipeline( data, @@ -27,17 +63,25 @@ export class FsBackupTransport implements BackupTransportInterface { // eslint-disable-next-line jsdoc/require-jsdoc retrieveData(backup: BackupEntity, resource: BackupResourceEntity): Readable | Promise { - const p: string = this.getPath(backup, resource); + const p: string = this.getResourcePath(backup, resource); return createReadStream(p); } // eslint-disable-next-line jsdoc/require-jsdoc async deleteData(backup: BackupEntity, resource: BackupResourceEntity): Promise { - const p: string = this.getPath(backup, resource); + const p: string = this.getResourcePath(backup, resource); await rm(p, { recursive: true }); } - private getPath(backup: BackupEntity, resource: BackupResourceEntity): string { - return path.join(this.backupBasePath, backup.name, resource.name); + private getResourcePath(backup: BackupEntity, resource: BackupResourceEntity): string { + return path.join(this.getBackupPath(backup), resource.name); + } + + private getBackupPath(backup: BackupEntity): string { + return path.join(this.backupBasePath, backup.name); + } + + private getBackupMetadataPath(backup: BackupEntity): string { + return path.join(this.getBackupPath(backup), METADATA_FILENAME); } } \ No newline at end of file diff --git a/src/data-source/data-source.service.ts b/src/data-source/data-source.service.ts index 40e92e0..0a503ef 100644 --- a/src/data-source/data-source.service.ts +++ b/src/data-source/data-source.service.ts @@ -8,6 +8,7 @@ import { Email, MailingList, MailingListSubscriber } from '../email'; import { BaseEntity } from '../entity/base-entity.model'; import { Log, LoggerInterface } from '../logging'; import { ThreadJobEntity } from '../multithreading'; +import { Invoice, NumberInvoices } from '../plugin'; import { Newable } from '../types'; import { validateEntitiesRegistered } from '../utilities'; import { WebsocketChannel, WebsocketMessage } from '../websocket'; @@ -35,7 +36,9 @@ export class DataSourceService implements DataSourceServiceInterface { MailingListSubscriber, Log, BackupResourceEntity, - BackupEntity + BackupEntity, + NumberInvoices, + Invoice ]; constructor() { diff --git a/src/plugin/invoicing/services/invoice-number.service.test.ts b/src/plugin/invoicing/services/invoice-number.service.test.ts index 4f5b5b8..e4ca09b 100644 --- a/src/plugin/invoicing/services/invoice-number.service.test.ts +++ b/src/plugin/invoicing/services/invoice-number.service.test.ts @@ -5,12 +5,12 @@ import { PostgreSqlContainer } from '@testcontainers/postgresql'; import { StartedTestContainer } from 'testcontainers'; import { InvoiceCalcService } from './invoice-calc.service'; +import { InvoiceNumberService } from './invoice-number.service'; +import { POSTGRES_TEST_IMAGE } from '../../../__testing__'; import { DataSource, MigrationEntity, Repository, PostgresDataSource, PostgresOptions } from '../../../data-source'; import { BaseEntity } from '../../../entity/base-entity.model'; import { Newable, OmitStrict } from '../../../types'; import { Invoice, InvoiceAddress, InvoicingOptions, NumberInvoices } from '../models'; -import { InvoiceNumberService } from './invoice-number.service'; -import { POSTGRES_TEST_IMAGE } from '../../../__testing__'; const currentYear: string = new Date() .getFullYear()