Release 2.1.2

package.json

@@ -1,6 +1,6 @@
 {
     "name": "zibri",
-    "version": "2.1.1",
+    "version": "2.1.2",
     "main": "./dist/index.js",
     "module": "./dist/index.mjs",
     "repository": {

sandbox/src/data-sources/db/db.data-source.ts

@@ -1,4 +1,4 @@
-import { PostgresDataSource, PostgresOptions, BaseEntity, DataSource, Newable, MigrationEntity, JwtRefreshToken, JwtCredentials, PasswordResetToken, MailingList, MailingListSubscriber, MailingListSubscriptionConfirmationToken, Log, Change, ChangeSet, Invoice, NumberInvoices, Entity, OmitClass, OtpCredentials, BackupResourceEntity, BackupEntity } from 'zibri';
+import { PostgresDataSource, PostgresOptions, BaseEntity, DataSource, Newable, MigrationEntity, JwtRefreshToken, JwtCredentials, PasswordResetToken, MailingList, MailingListSubscriber, MailingListSubscriptionConfirmationToken, Log, Change, ChangeSet, Entity, OmitClass, OtpCredentials, BackupResourceEntity, BackupEntity, Invoice, NumberInvoices } from 'zibri';
 
 import { Company, Test, User } from '../../models';
 

sandbox/src/index.ts

@@ -2,10 +2,10 @@ import H from 'handlebars/runtime';
 import { inject, isVersion, JwtAuthController, LoggerInterface, ZIBRI_DI_TOKENS, ZibriApplication, ZibriInvoicingPlugin } from 'zibri';
 
 import { CronController, FileController, MetricsController, TemplateController, TestController, TestCrudController, TestWebsocketController } from './controllers';
-import { DbDataSource } from './data-sources';
-import { version } from '../package.json';
 import { createDefaultData } from './create-default-data.function';
 import { StatusCronJob } from './cron';
+import { DbDataSource } from './data-sources';
+import { version } from '../package.json';
 import { providers } from './providers';
 
 export let logger: LoggerInterface;

src/backup/backup-entity.model.ts

@@ -1,9 +1,9 @@
 import { inject, ZIBRI_DI_TOKENS } from '../di';
 import { Entity, Property } from '../entity';
+import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model';
 import { BaseEntity } from '../entity/base-entity.model';
 import { FormatDateFn } from '../localization';
 import { OmitStrict } from '../types';
-import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model';
 
 /**
  * The entity of a single backup.

src/backup/backup-resource-entity.model.ts

@@ -1,7 +1,7 @@
 import { Entity, Property } from '../entity';
+import { BackupEntity } from './backup-entity.model';
 import { BaseEntity } from '../entity/base-entity.model';
 import { OmitStrict } from '../types';
-import { BackupEntity } from './backup-entity.model';
 
 /**
  * A single resource entity.

src/backup/backup-service.interface.ts

@@ -29,6 +29,10 @@ export interface BackupServiceInterface {
      * Initializes the service.
      */
     init: () => void | Promise<void>,
+    /**
+     * Synchronizes backup entities with the data from all transports.
+     */
+    syncBackupEntities: () => Promise<void>,
     /**
      * Starts a new backup based on the given configuration.
      */

src/backup/backup-service.test.ts

@@ -89,6 +89,10 @@ describe('Create and restore postgres backup', () => {
         expect((await itemRepository.findAll()).length).toEqual(1);
         await itemRepository.deleteAll({});
         expect((await itemRepository.findAll()).length).toEqual(0);
+        await backupRepository.deleteAll({});
+        expect((await backupRepository.findAll()).length).toEqual(0);
+        await backupService.syncBackupEntities();
+        expect((await backupRepository.findAll()).length).toEqual(1);
         const backup: BackupEntity = (await backupRepository.findAll({ relations: ['resources'] }))[0];
         await backupService.restore(backup);
         expect((await itemRepository.findAll()).length).toEqual(1);

src/backup/backup.service.ts

@@ -7,7 +7,7 @@ import { inject, repositoryTokenFor, ZIBRI_DI_TOKENS } from '../di';
 import { GlobalRegistry } from '../global';
 import { LoggerInterface } from '../logging';
 import { Newable } from '../types';
-import { MetadataUtilities, validateEntitiesRegistered } from '../utilities';
+import { chunkedPromiseAll, MetadataUtilities, validateEntitiesRegistered } from '../utilities';
 import { BackupEntity, BackupEntityCreateData } from './backup-entity.model';
 import { BackupResourceEntity, BackupResourceEntityCreateData } from './backup-resource-entity.model';
 import { BackupResourceInterface } from './backup-resource.interface';
@@ -58,6 +58,62 @@ export class BackupService implements BackupServiceInterface {
             }
             await this.logger.info(`  - ${resourceClass.name}`);
         }
+
+        await this.syncBackupEntities();
+    }
+
+    // eslint-disable-next-line jsdoc/require-jsdoc
+    async syncBackupEntities(): Promise<void> {
+        if (!this.backupResources.length) {
+            return;
+        }
+        const transports: BackupTransportInterface[] = this.backupResources
+            .map(r => MetadataUtilities.getBackupResourceMetadata(r)?.transports ?? [])
+            .flat();
+        if (!transports.length) {
+            await this.logger.warn('Could not find any transports to sync backup entities from');
+            return;
+        }
+        const existingEntities: BackupEntity[] = await this.backupRepository.findAll();
+        const resolvedEntities: BackupEntity[] = (await Promise.all(transports.map(t => t.resolveBackups(existingEntities)))).flat();
+        const entitiesToSync: BackupEntity[] = resolvedEntities.filter(entity => !existingEntities.map(e => e.id).includes(entity.id));
+
+        const groupedEntities: Record<string, BackupEntity[]> = this.groupEntitiesById(entitiesToSync);
+        const mergedEntities: BackupEntity[] = this.mergeEntities(groupedEntities);
+        await chunkedPromiseAll(mergedEntities.map(e => this.backupRepository.create(e, { allowId: true })));
+    }
+
+    private mergeEntities(groupedEntities: Record<string, BackupEntity[]>): BackupEntity[] {
+        const res: BackupEntity[] = [];
+        for (const key in groupedEntities) {
+            const entities: BackupEntity[] = groupedEntities[key];
+            res.push({
+                createdAt: entities[0].createdAt,
+                id: entities[0].id,
+                name: entities[0].name,
+                resources: entities.reduce<BackupResourceEntity[]>(
+                    (prev, curr) => [
+                        ...prev,
+                        ...curr.resources.filter(r => !prev.find(pr => pr.id === r.id))
+                    ],
+                    []
+                )
+            });
+        }
+        return res;
+    }
+
+    private groupEntitiesById(entities: BackupEntity[]): Record<string, BackupEntity[]> {
+        const res: Record<string, BackupEntity[]> = {};
+        for (const entity of entities) {
+            if (res[entity.id] == undefined) {
+                res[entity.id] = [entity];
+            }
+            else {
+                res[entity.id].push(entity);
+            }
+        }
+        return res;
     }
 
     // eslint-disable-next-line jsdoc/require-jsdoc

src/backup/decorators/backup-resource.decorator.ts

@@ -1,10 +1,10 @@
 
+import { BackupResourceMetadata, BackupResourceMetadataInput } from './backup-resource-metadata.model';
 import { DiToken } from '../../di';
 import { GlobalRegistry } from '../../global';
 import { Newable } from '../../types';
 import { MetadataUtilities } from '../../utilities';
 import { BackupResourceInterface } from '../backup-resource.interface';
-import { BackupResourceMetadata, BackupResourceMetadataInput } from './backup-resource-metadata.model';
 
 // eslint-disable-next-line jsdoc/require-returns
 /**

src/backup/transports/backup-transport.interface.ts

@@ -11,6 +11,10 @@ export interface BackupTransportInterface {
      * The name of the transport. Is used to resolve the transport when restoring backups, so this should not change.
      */
     readonly name: string,
+    /**
+     * Resolves all backups that have data stored over this transport.
+     */
+    resolveBackups: (existingEntities: BackupEntity[]) => BackupEntity[] | Promise<BackupEntity[]>,
     /**
      * Stores the given data somewhere.
      */

src/backup/transports/fs.backup-transport.ts

@@ -1,23 +1,59 @@
-import { createReadStream, createWriteStream } from 'fs';
-import { mkdir, rm } from 'fs/promises';
+import { createReadStream, createWriteStream, Dirent } from 'fs';
+import { mkdir, readdir, readFile, rm, writeFile } from 'fs/promises';
 import path from 'path';
 import { Readable } from 'stream';
 import { pipeline } from 'stream/promises';
 
 import { BackupTransportInterface } from './backup-transport.interface';
+import { inject, ZIBRI_DI_TOKENS } from '../../di';
+import { LoggerInterface } from '../../logging';
+import { pathExists } from '../../utilities';
 import { BackupEntity } from '../backup-entity.model';
 import { BackupResourceEntity } from '../backup-resource-entity.model';
 
+const METADATA_FILENAME: string = 'metadata.json';
+
 /**
  * A transport that saves the data to the local file system.
  */
 export class FsBackupTransport implements BackupTransportInterface {
+
+    private get logger(): LoggerInterface {
+        return inject(ZIBRI_DI_TOKENS.LOGGER);
+    }
+
     constructor(readonly name: string, protected readonly backupBasePath: string) {}
 
+    // eslint-disable-next-line jsdoc/require-jsdoc
+    async resolveBackups(existingEntities: BackupEntity[]): Promise<BackupEntity[]> {
+        if (!await pathExists(this.backupBasePath)) {
+            return [];
+        }
+
+        const nodes: Dirent<string>[] = await readdir(this.backupBasePath, { withFileTypes: true });
+        const res: BackupEntity[] = (await Promise.all(nodes.map(async node => {
+            if (!node.isDirectory()) {
+                return;
+            }
+            if (existingEntities.map(e => e.name).includes(node.name)) {
+                return;
+            }
+            const p: string = path.join(node.parentPath, node.name, METADATA_FILENAME);
+            if (!await pathExists(p)) {
+                await this.logger.warn(`Could not find the metadata file needed to resolve a backup: "${p}"`);
+                return;
+            }
+            const json: string = await readFile(p, 'utf8');
+            return JSON.parse(json) as BackupEntity;
+        }))).filter(b => b != undefined);
+        return res;
+    }
+
     // eslint-disable-next-line jsdoc/require-jsdoc
     async storeData(data: Readable, backup: BackupEntity, resource: BackupResourceEntity): Promise<void> {
-        const p: string = this.getPath(backup, resource);
-        await mkdir(path.join(this.backupBasePath, backup.name), { recursive: true });
+        const p: string = this.getResourcePath(backup, resource);
+        await mkdir(this.getBackupPath(backup), { recursive: true });
+        await writeFile(this.getBackupMetadataPath(backup), JSON.stringify(backup), 'utf8');
 
         await pipeline(
             data,
@@ -27,17 +63,25 @@ export class FsBackupTransport implements BackupTransportInterface {
 
     // eslint-disable-next-line jsdoc/require-jsdoc
     retrieveData(backup: BackupEntity, resource: BackupResourceEntity): Readable | Promise<Readable> {
-        const p: string = this.getPath(backup, resource);
+        const p: string = this.getResourcePath(backup, resource);
         return createReadStream(p);
     }
 
     // eslint-disable-next-line jsdoc/require-jsdoc
     async deleteData(backup: BackupEntity, resource: BackupResourceEntity): Promise<void> {
-        const p: string = this.getPath(backup, resource);
+        const p: string = this.getResourcePath(backup, resource);
         await rm(p, { recursive: true });
     }
 
-    private getPath(backup: BackupEntity, resource: BackupResourceEntity): string {
-        return path.join(this.backupBasePath, backup.name, resource.name);
+    private getResourcePath(backup: BackupEntity, resource: BackupResourceEntity): string {
+        return path.join(this.getBackupPath(backup), resource.name);
+    }
+
+    private getBackupPath(backup: BackupEntity): string {
+        return path.join(this.backupBasePath, backup.name);
+    }
+
+    private getBackupMetadataPath(backup: BackupEntity): string {
+        return path.join(this.getBackupPath(backup), METADATA_FILENAME);
     }
 }

src/data-source/data-source.service.ts

@@ -8,6 +8,7 @@ import { Email, MailingList, MailingListSubscriber } from '../email';
 import { BaseEntity } from '../entity/base-entity.model';
 import { Log, LoggerInterface } from '../logging';
 import { ThreadJobEntity } from '../multithreading';
+import { Invoice, NumberInvoices } from '../plugin';
 import { Newable } from '../types';
 import { validateEntitiesRegistered } from '../utilities';
 import { WebsocketChannel, WebsocketMessage } from '../websocket';
@@ -35,7 +36,9 @@ export class DataSourceService implements DataSourceServiceInterface {
         MailingListSubscriber,
         Log,
         BackupResourceEntity,
-        BackupEntity
+        BackupEntity,
+        NumberInvoices,
+        Invoice
     ];
 
     constructor() {

src/plugin/invoicing/services/invoice-number.service.test.ts

@@ -5,12 +5,12 @@ import { PostgreSqlContainer } from '@testcontainers/postgresql';
 import { StartedTestContainer } from 'testcontainers';
 
 import { InvoiceCalcService } from './invoice-calc.service';
+import { InvoiceNumberService } from './invoice-number.service';
+import { POSTGRES_TEST_IMAGE } from '../../../__testing__';
 import { DataSource, MigrationEntity, Repository, PostgresDataSource, PostgresOptions } from '../../../data-source';
 import { BaseEntity } from '../../../entity/base-entity.model';
 import { Newable, OmitStrict } from '../../../types';
 import { Invoice, InvoiceAddress, InvoicingOptions, NumberInvoices } from '../models';
-import { InvoiceNumberService } from './invoice-number.service';
-import { POSTGRES_TEST_IMAGE } from '../../../__testing__';
 
 const currentYear: string = new Date()
     .getFullYear()