diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index a76124fde..97dd6939b 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -43,6 +43,7 @@ import { DashboardModule } from './entities/visualizations/dashboard/dashboards. import { PanelModule } from './entities/visualizations/panel/panel.module.js'; import { PanelPositionModule } from './entities/visualizations/panel-position/panel-position.module.js'; import { TableWidgetModule } from './entities/widget/table-widget.module.js'; +import { AgentsModule } from './microservices/agents-microservice/agents.module.js'; import { SaaSGatewayModule } from './microservices/gateways/saas-gateway.ts/saas-gateway.module.js'; import { SaasModule } from './microservices/saas-microservice/saas.module.js'; import { AppLoggerMiddleware } from './middlewares/logging-middleware/app-logger-middlewate.js'; @@ -84,6 +85,7 @@ import { GetHelloUseCase } from './use-cases-app/get-hello.use.case.js'; SharedModule, TableActionModule, SaasModule, + AgentsModule, CompanyInfoModule, SaaSGatewayModule, TableTriggersModule, diff --git a/backend/src/common/data-injection.tokens.ts b/backend/src/common/data-injection.tokens.ts index c8817a7e8..a6d58c8a6 100644 --- a/backend/src/common/data-injection.tokens.ts +++ b/backend/src/common/data-injection.tokens.ts @@ -177,6 +177,15 @@ export enum UseCaseType { FIND_USER_AI_CHAT_BY_ID = 'FIND_USER_AI_CHAT_BY_ID', DELETE_USER_AI_CHAT = 'DELETE_USER_AI_CHAT', + AGENTS_VALIDATE_USER_TOKEN = 'AGENTS_VALIDATE_USER_TOKEN', + AGENTS_VALIDATE_TABLE_AI_REQUEST = 'AGENTS_VALIDATE_TABLE_AI_REQUEST', + AGENTS_VALIDATE_CONNECTION_EDIT = 'AGENTS_VALIDATE_CONNECTION_EDIT', + AGENTS_GET_AI_CONNECTION_CONTEXT = 'AGENTS_GET_AI_CONNECTION_CONTEXT', + AGENTS_GET_AI_TABLE_STRUCTURE = 'AGENTS_GET_AI_TABLE_STRUCTURE', + AGENTS_EXECUTE_AI_RAW_QUERY = 'AGENTS_EXECUTE_AI_RAW_QUERY', + AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE = 'AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE', + AGENTS_SCAN_AND_CREATE_SETTINGS = 'AGENTS_SCAN_AND_CREATE_SETTINGS', + CREATE_TABLE_FILTERS = 'CREATE_TABLE_FILTERS', FIND_TABLE_FILTERS = 'FIND_TABLE_FILTERS', DELETE_TABLE_FILTERS = 'DELETE_TABLE_FILTERS', diff --git a/backend/src/entities/ai/ai-conversation-history/user-ai-chat.controller.ts b/backend/src/entities/ai/ai-conversation-history/user-ai-chat.controller.ts index 37b78e35c..6d3df02ad 100644 --- a/backend/src/entities/ai/ai-conversation-history/user-ai-chat.controller.ts +++ b/backend/src/entities/ai/ai-conversation-history/user-ai-chat.controller.ts @@ -35,7 +35,12 @@ export class UserAiChatController { private readonly deleteUserAiChatUseCase: IDeleteUserAiChat, ) {} - @ApiOperation({ summary: 'Get all AI chats for current user' }) + @ApiOperation({ + summary: 'Get all AI chats for current user', + deprecated: true, + description: + 'Deprecated: this endpoint moved to the agents microservice (same path and contract). It remains here only until traffic is switched over.', + }) @ApiResponse({ status: 200, description: 'Returns list of AI chats.', @@ -47,7 +52,12 @@ export class UserAiChatController { return await this.findUserAiChatsUseCase.execute(inputData, InTransactionEnum.OFF); } - @ApiOperation({ summary: 'Get AI chat by ID with all messages' }) + @ApiOperation({ + summary: 'Get AI chat by ID with all messages', + deprecated: true, + description: + 'Deprecated: this endpoint moved to the agents microservice (same path and contract). It remains here only until traffic is switched over.', + }) @ApiResponse({ status: 200, description: 'Returns AI chat with messages.', @@ -60,7 +70,12 @@ export class UserAiChatController { return await this.findUserAiChatByIdUseCase.execute(inputData, InTransactionEnum.OFF); } - @ApiOperation({ summary: 'Delete AI chat by ID' }) + @ApiOperation({ + summary: 'Delete AI chat by ID', + deprecated: true, + description: + 'Deprecated: this endpoint moved to the agents microservice (same path and contract). It remains here only until traffic is switched over.', + }) @ApiResponse({ status: 200, description: 'AI chat deleted successfully.', diff --git a/backend/src/entities/ai/user-ai-requests-v2.controller.ts b/backend/src/entities/ai/user-ai-requests-v2.controller.ts index 281419d07..30d61fd44 100644 --- a/backend/src/entities/ai/user-ai-requests-v2.controller.ts +++ b/backend/src/entities/ai/user-ai-requests-v2.controller.ts @@ -44,6 +44,9 @@ export class UserAIRequestsControllerV2 { @ApiOperation({ summary: 'Request info from table in connection with AI with conversation history (Version 4)', + deprecated: true, + description: + 'Deprecated: this endpoint moved to the agents microservice (same path and contract). It remains here only until traffic is switched over.', }) @ApiResponse({ status: 201, @@ -86,6 +89,9 @@ export class UserAIRequestsControllerV2 { @ApiOperation({ summary: 'Request AI settings and widgets creation for connection', + deprecated: true, + description: + 'Deprecated: this endpoint moved to the agents microservice (same path and contract). It remains here only until traffic is switched over.', }) @ApiResponse({ status: 200, diff --git a/backend/src/microservices/agents-microservice/agents.controller.ts b/backend/src/microservices/agents-microservice/agents.controller.ts new file mode 100644 index 000000000..52b1ee948 --- /dev/null +++ b/backend/src/microservices/agents-microservice/agents.controller.ts @@ -0,0 +1,187 @@ +import { Body, Controller, Inject, Injectable, Post, Res, UseInterceptors } from '@nestjs/common'; +import { ApiBody, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; +import { SkipThrottle } from '@nestjs/throttler'; +import { Response } from 'express'; +import { UseCaseType } from '../../common/data-injection.tokens.js'; +import { SlugUuid } from '../../decorators/slug-uuid.decorator.js'; +import { Timeout, TimeoutDefaults } from '../../decorators/timeout.decorator.js'; +import { InTransactionEnum } from '../../enums/in-transaction.enum.js'; +import { isTest } from '../../helpers/app/is-test.js'; +import { SentryInterceptor } from '../../interceptors/sentry.interceptor.js'; +import { + AiConnectionContextRO, + AiQueryResultRO, + PermissionAllowedRO, + ValidatedUserTokenRO, +} from './data-structures/agents-responses.ds.js'; +import { + AiDataRequestBaseDto, + ExecuteAiAggregationPipelineDto, + ExecuteAiRawQueryDto, + GetAiTableStructureDto, +} from './dto/agents-ai-data.dtos.js'; +import { ValidateConnectionEditDto, ValidateTableAiRequestDto, ValidateUserTokenDto } from './dto/agents-auth.dtos.js'; +import { + IExecuteAiAggregationPipeline, + IExecuteAiRawQuery, + IGetAiConnectionContext, + IGetAiTableStructure, + IScanAndCreateSettings, + IValidateConnectionEdit, + IValidateTableAiRequest, + IValidateUserToken, +} from './use-cases/agents-use-cases.interface.js'; + +@UseInterceptors(SentryInterceptor) +@SkipThrottle() +@Timeout() +@ApiTags('agents microservice') +@Controller('internal/agents') +@Injectable() +export class AgentsController { + constructor( + @Inject(UseCaseType.AGENTS_VALIDATE_USER_TOKEN) + private readonly validateUserTokenUseCase: IValidateUserToken, + @Inject(UseCaseType.AGENTS_VALIDATE_TABLE_AI_REQUEST) + private readonly validateTableAiRequestUseCase: IValidateTableAiRequest, + @Inject(UseCaseType.AGENTS_VALIDATE_CONNECTION_EDIT) + private readonly validateConnectionEditUseCase: IValidateConnectionEdit, + @Inject(UseCaseType.AGENTS_GET_AI_CONNECTION_CONTEXT) + private readonly getAiConnectionContextUseCase: IGetAiConnectionContext, + @Inject(UseCaseType.AGENTS_GET_AI_TABLE_STRUCTURE) + private readonly getAiTableStructureUseCase: IGetAiTableStructure, + @Inject(UseCaseType.AGENTS_EXECUTE_AI_RAW_QUERY) + private readonly executeAiRawQueryUseCase: IExecuteAiRawQuery, + @Inject(UseCaseType.AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE) + private readonly executeAiAggregationPipelineUseCase: IExecuteAiAggregationPipeline, + @Inject(UseCaseType.AGENTS_SCAN_AND_CREATE_SETTINGS) + private readonly scanAndCreateSettingsUseCase: IScanAndCreateSettings, + ) {} + + @ApiOperation({ summary: 'Validate an end-user JWT on behalf of the agents microservice' }) + @ApiResponse({ status: 201, type: ValidatedUserTokenRO }) + @ApiBody({ type: ValidateUserTokenDto }) + @Post('/auth/validate-user-token') + public async validateUserToken(@Body() body: ValidateUserTokenDto): Promise { + return await this.validateUserTokenUseCase.execute(body.token, InTransactionEnum.OFF); + } + + @ApiOperation({ summary: 'Check Cedar permission for an AI request on a table' }) + @ApiResponse({ status: 201, type: PermissionAllowedRO }) + @ApiBody({ type: ValidateTableAiRequestDto }) + @Post('/auth/validate-table-ai-request') + public async validateTableAiRequest(@Body() body: ValidateTableAiRequestDto): Promise { + return await this.validateTableAiRequestUseCase.execute( + { userId: body.userId, connectionId: body.connectionId, tableName: body.tableName }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Check Cedar permission for editing a connection' }) + @ApiResponse({ status: 201, type: PermissionAllowedRO }) + @ApiBody({ type: ValidateConnectionEditDto }) + @Post('/auth/validate-connection-edit') + public async validateConnectionEdit(@Body() body: ValidateConnectionEditDto): Promise { + return await this.validateConnectionEditUseCase.execute( + { userId: body.userId, connectionId: body.connectionId }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Get AI-relevant connection context (type, schema, MongoDB flag)' }) + @ApiResponse({ status: 201, type: AiConnectionContextRO }) + @ApiBody({ type: AiDataRequestBaseDto }) + @Post('/ai/data/:connectionId/context') + public async getAiConnectionContext( + @SlugUuid('connectionId') connectionId: string, + @Body() body: AiDataRequestBaseDto, + ): Promise { + return await this.getAiConnectionContextUseCase.execute( + { connectionId, userId: body.userId, masterPassword: body.masterPassword ?? null }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Get permission-aware table structure for the AI tool loop' }) + @ApiResponse({ status: 201, description: 'Table structure with related tables.' }) + @ApiBody({ type: GetAiTableStructureDto }) + @Timeout(!isTest() ? TimeoutDefaults.EXTENDED : TimeoutDefaults.EXTENDED_TEST) + @Post('/ai/data/:connectionId/table-structure') + public async getAiTableStructure( + @SlugUuid('connectionId') connectionId: string, + @Body() body: GetAiTableStructureDto, + ): Promise> { + return await this.getAiTableStructureUseCase.execute( + { + connectionId, + userId: body.userId, + masterPassword: body.masterPassword ?? null, + tableName: body.tableName, + }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Validate and execute a read-only SQL query for the AI tool loop' }) + @ApiResponse({ status: 201, type: AiQueryResultRO }) + @ApiBody({ type: ExecuteAiRawQueryDto }) + @Timeout(!isTest() ? TimeoutDefaults.EXTENDED : TimeoutDefaults.EXTENDED_TEST) + @Post('/ai/data/:connectionId/raw-query') + public async executeAiRawQuery( + @SlugUuid('connectionId') connectionId: string, + @Body() body: ExecuteAiRawQueryDto, + ): Promise { + return await this.executeAiRawQueryUseCase.execute( + { + connectionId, + userId: body.userId, + masterPassword: body.masterPassword ?? null, + tableName: body.tableName, + query: body.query, + }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Validate and execute a read-only MongoDB aggregation pipeline for the AI tool loop' }) + @ApiResponse({ status: 201, type: AiQueryResultRO }) + @ApiBody({ type: ExecuteAiAggregationPipelineDto }) + @Timeout(!isTest() ? TimeoutDefaults.EXTENDED : TimeoutDefaults.EXTENDED_TEST) + @Post('/ai/data/:connectionId/aggregation-pipeline') + public async executeAiAggregationPipeline( + @SlugUuid('connectionId') connectionId: string, + @Body() body: ExecuteAiAggregationPipelineDto, + ): Promise { + return await this.executeAiAggregationPipelineUseCase.execute( + { + connectionId, + userId: body.userId, + masterPassword: body.masterPassword ?? null, + tableName: body.tableName, + pipeline: body.pipeline, + }, + InTransactionEnum.OFF, + ); + } + + @ApiOperation({ summary: 'Run the AI settings/widgets scan, streaming progress chunks' }) + @ApiResponse({ status: 201, description: 'Streams progress as newline-delimited JSON chunks.' }) + @ApiBody({ type: AiDataRequestBaseDto }) + @Timeout(!isTest() ? TimeoutDefaults.AI : TimeoutDefaults.AI_TEST) + @Post('/ai/data/:connectionId/settings-scan') + public async scanAndCreateSettings( + @SlugUuid('connectionId') connectionId: string, + @Body() body: AiDataRequestBaseDto, + @Res({ passthrough: true }) response: Response, + ): Promise { + return await this.scanAndCreateSettingsUseCase.execute( + { + connectionId, + userId: body.userId, + masterPassword: body.masterPassword ?? null, + response, + }, + InTransactionEnum.OFF, + ); + } +} diff --git a/backend/src/microservices/agents-microservice/agents.module.ts b/backend/src/microservices/agents-microservice/agents.module.ts new file mode 100644 index 000000000..73b7740c6 --- /dev/null +++ b/backend/src/microservices/agents-microservice/agents.module.ts @@ -0,0 +1,62 @@ +import { MiddlewareConsumer, Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { SaaSAuthMiddleware } from '../../authorization/saas-auth.middleware.js'; +import { GlobalDatabaseContext } from '../../common/application/global-database-context.js'; +import { BaseType, UseCaseType } from '../../common/data-injection.tokens.js'; +import { AgentsController } from './agents.controller.js'; +import { ExecuteAiAggregationPipelineUseCase } from './use-cases/execute-ai-aggregation-pipeline.use.case.js'; +import { ExecuteAiRawQueryUseCase } from './use-cases/execute-ai-raw-query.use.case.js'; +import { GetAiConnectionContextUseCase } from './use-cases/get-ai-connection-context.use.case.js'; +import { GetAiTableStructureUseCase } from './use-cases/get-ai-table-structure.use.case.js'; +import { ScanAndCreateSettingsUseCase } from './use-cases/scan-and-create-settings.use.case.js'; +import { ValidateConnectionEditUseCase } from './use-cases/validate-connection-edit.use.case.js'; +import { ValidateTableAiRequestUseCase } from './use-cases/validate-table-ai-request.use.case.js'; +import { ValidateUserTokenUseCase } from './use-cases/validate-user-token.use.case.js'; + +@Module({ + imports: [TypeOrmModule.forFeature([])], + providers: [ + { + provide: BaseType.GLOBAL_DB_CONTEXT, + useClass: GlobalDatabaseContext, + }, + { + provide: UseCaseType.AGENTS_VALIDATE_USER_TOKEN, + useClass: ValidateUserTokenUseCase, + }, + { + provide: UseCaseType.AGENTS_VALIDATE_TABLE_AI_REQUEST, + useClass: ValidateTableAiRequestUseCase, + }, + { + provide: UseCaseType.AGENTS_VALIDATE_CONNECTION_EDIT, + useClass: ValidateConnectionEditUseCase, + }, + { + provide: UseCaseType.AGENTS_GET_AI_CONNECTION_CONTEXT, + useClass: GetAiConnectionContextUseCase, + }, + { + provide: UseCaseType.AGENTS_GET_AI_TABLE_STRUCTURE, + useClass: GetAiTableStructureUseCase, + }, + { + provide: UseCaseType.AGENTS_EXECUTE_AI_RAW_QUERY, + useClass: ExecuteAiRawQueryUseCase, + }, + { + provide: UseCaseType.AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE, + useClass: ExecuteAiAggregationPipelineUseCase, + }, + { + provide: UseCaseType.AGENTS_SCAN_AND_CREATE_SETTINGS, + useClass: ScanAndCreateSettingsUseCase, + }, + ], + controllers: [AgentsController], +}) +export class AgentsModule { + public configure(consumer: MiddlewareConsumer): void { + consumer.apply(SaaSAuthMiddleware).forRoutes(AgentsController); + } +} diff --git a/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts b/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts new file mode 100644 index 000000000..36bf5072e --- /dev/null +++ b/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts @@ -0,0 +1,42 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class ValidatedUserTokenRO { + @ApiProperty() + sub: string; + + @ApiPropertyOptional() + email: string | null; + + @ApiPropertyOptional() + exp: number | null; + + @ApiPropertyOptional() + iat: number | null; +} + +export class PermissionAllowedRO { + @ApiProperty() + allowed: boolean; +} + +export class AiConnectionContextRO { + @ApiProperty() + connectionId: string; + + @ApiProperty() + type: string; + + @ApiPropertyOptional() + schema: string | null; + + @ApiProperty() + isMongoDb: boolean; + + @ApiProperty() + userEmail: string; +} + +export class AiQueryResultRO { + @ApiProperty() + result: unknown; +} diff --git a/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts b/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts new file mode 100644 index 000000000..cfc4339ee --- /dev/null +++ b/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts @@ -0,0 +1,36 @@ +import { Response } from 'express'; + +export class ValidateTableAiRequestDs { + userId: string; + connectionId: string; + tableName: string; +} + +export class ValidateConnectionEditDs { + userId: string; + connectionId: string; +} + +export class AiDataRequestDs { + connectionId: string; + userId: string; + masterPassword: string | null; +} + +export class GetAiTableStructureDs extends AiDataRequestDs { + tableName: string; +} + +export class ExecuteAiRawQueryDs extends AiDataRequestDs { + tableName: string; + query: string; +} + +export class ExecuteAiAggregationPipelineDs extends AiDataRequestDs { + tableName: string; + pipeline: string; +} + +export class ScanAndCreateSettingsDs extends AiDataRequestDs { + response: Response; +} diff --git a/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts b/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts new file mode 100644 index 000000000..5f79edec5 --- /dev/null +++ b/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts @@ -0,0 +1,46 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsNotEmpty, IsOptional, IsString, IsUUID } from 'class-validator'; + +export class AiDataRequestBaseDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + @IsUUID() + userId: string; + + @ApiPropertyOptional() + @IsOptional() + @IsString() + masterPassword?: string | null; +} + +export class GetAiTableStructureDto extends AiDataRequestBaseDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + tableName: string; +} + +export class ExecuteAiRawQueryDto extends AiDataRequestBaseDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + tableName: string; + + @ApiProperty() + @IsString() + @IsNotEmpty() + query: string; +} + +export class ExecuteAiAggregationPipelineDto extends AiDataRequestBaseDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + tableName: string; + + @ApiProperty() + @IsString() + @IsNotEmpty() + pipeline: string; +} diff --git a/backend/src/microservices/agents-microservice/dto/agents-auth.dtos.ts b/backend/src/microservices/agents-microservice/dto/agents-auth.dtos.ts new file mode 100644 index 000000000..8825ef566 --- /dev/null +++ b/backend/src/microservices/agents-microservice/dto/agents-auth.dtos.ts @@ -0,0 +1,38 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class ValidateUserTokenDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + token: string; +} + +export class ValidateTableAiRequestDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + userId: string; + + @ApiProperty() + @IsString() + @IsNotEmpty() + connectionId: string; + + @ApiProperty() + @IsString() + @IsNotEmpty() + tableName: string; +} + +export class ValidateConnectionEditDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + userId: string; + + @ApiProperty() + @IsString() + @IsNotEmpty() + connectionId: string; +} diff --git a/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts b/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts new file mode 100644 index 000000000..2e03408e2 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts @@ -0,0 +1,48 @@ +import { InTransactionEnum } from '../../../enums/in-transaction.enum.js'; +import { + AiDataRequestDs, + ExecuteAiAggregationPipelineDs, + ExecuteAiRawQueryDs, + GetAiTableStructureDs, + ScanAndCreateSettingsDs, + ValidateConnectionEditDs, + ValidateTableAiRequestDs, +} from '../data-structures/agents.ds.js'; +import { + AiConnectionContextRO, + AiQueryResultRO, + PermissionAllowedRO, + ValidatedUserTokenRO, +} from '../data-structures/agents-responses.ds.js'; + +export interface IValidateUserToken { + execute(token: string, inTransaction: InTransactionEnum): Promise; +} + +export interface IValidateTableAiRequest { + execute(inputData: ValidateTableAiRequestDs, inTransaction: InTransactionEnum): Promise; +} + +export interface IValidateConnectionEdit { + execute(inputData: ValidateConnectionEditDs, inTransaction: InTransactionEnum): Promise; +} + +export interface IGetAiConnectionContext { + execute(inputData: AiDataRequestDs, inTransaction: InTransactionEnum): Promise; +} + +export interface IGetAiTableStructure { + execute(inputData: GetAiTableStructureDs, inTransaction: InTransactionEnum): Promise>; +} + +export interface IExecuteAiRawQuery { + execute(inputData: ExecuteAiRawQueryDs, inTransaction: InTransactionEnum): Promise; +} + +export interface IExecuteAiAggregationPipeline { + execute(inputData: ExecuteAiAggregationPipelineDs, inTransaction: InTransactionEnum): Promise; +} + +export interface IScanAndCreateSettings { + execute(inputData: ScanAndCreateSettingsDs, inTransaction: InTransactionEnum): Promise; +} diff --git a/backend/src/microservices/agents-microservice/use-cases/execute-ai-aggregation-pipeline.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/execute-ai-aggregation-pipeline.use.case.ts new file mode 100644 index 000000000..25445744a --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/execute-ai-aggregation-pipeline.use.case.ts @@ -0,0 +1,61 @@ +import { BadRequestException, Inject, Injectable, Scope } from '@nestjs/common'; +import { isReadOnlyMongoAggregationPipeline, isValidMongoDbCommand } from '../../../ai-core/tools/query-validators.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarPermissionsService } from '../../../entities/cedar-authorization/cedar-permissions.service.js'; +import { ExecuteAiAggregationPipelineDs } from '../data-structures/agents.ds.js'; +import { AiQueryResultRO } from '../data-structures/agents-responses.ds.js'; +import { assertUserCanReadPipelineCollections, setupAiConnection } from '../utils/ai-data-access.helpers.js'; +import { IExecuteAiAggregationPipeline } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ExecuteAiAggregationPipelineUseCase + extends AbstractUseCase + implements IExecuteAiAggregationPipeline +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarPermissions: CedarPermissionsService, + ) { + super(); + } + + protected async implementation(inputData: ExecuteAiAggregationPipelineDs): Promise { + const { connectionId, userId, masterPassword, tableName, pipeline } = inputData; + + const { foundConnection, dataAccessObject, userEmail } = await setupAiConnection( + this._dbContext, + connectionId, + masterPassword, + userId, + ); + + if (!isValidMongoDbCommand(pipeline)) { + throw new BadRequestException( + 'Invalid MongoDB command. Please ensure it is a read-only aggregation pipeline without any forbidden keywords.', + ); + } + + if (!isReadOnlyMongoAggregationPipeline(pipeline)) { + throw new BadRequestException( + 'Invalid MongoDB command. Aggregation stages that write data ($out, $merge) or execute ' + + 'server-side JavaScript ($function, $accumulator, $where) are not allowed.', + ); + } + + await assertUserCanReadPipelineCollections( + this.cedarPermissions, + pipeline, + tableName, + userId, + foundConnection.id, + dataAccessObject, + ); + + const pipelineResult = await dataAccessObject.executeRawQuery(pipeline, tableName, userEmail); + + return { result: pipelineResult }; + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/execute-ai-raw-query.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/execute-ai-raw-query.use.case.ts new file mode 100644 index 000000000..d5f90bf57 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/execute-ai-raw-query.use.case.ts @@ -0,0 +1,57 @@ +import { BadRequestException, Inject, Injectable, Scope } from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { isValidSQLQuery, wrapQueryWithLimit } from '../../../ai-core/tools/query-validators.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarPermissionsService } from '../../../entities/cedar-authorization/cedar-permissions.service.js'; +import { assertUserCanReadQueryTables } from '../../../entities/visualizations/panel/utils/assert-query-tables-readable.util.js'; +import { ExecuteAiRawQueryDs } from '../data-structures/agents.ds.js'; +import { AiQueryResultRO } from '../data-structures/agents-responses.ds.js'; +import { setupAiConnection } from '../utils/ai-data-access.helpers.js'; +import { IExecuteAiRawQuery } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ExecuteAiRawQueryUseCase + extends AbstractUseCase + implements IExecuteAiRawQuery +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarPermissions: CedarPermissionsService, + ) { + super(); + } + + protected async implementation(inputData: ExecuteAiRawQueryDs): Promise { + const { connectionId, userId, masterPassword, tableName, query } = inputData; + + const { foundConnection, dataAccessObject, userEmail } = await setupAiConnection( + this._dbContext, + connectionId, + masterPassword, + userId, + ); + + if (!isValidSQLQuery(query)) { + throw new BadRequestException( + 'Invalid SQL query. Please ensure it is a read-only SELECT statement without any forbidden keywords.', + ); + } + + await assertUserCanReadQueryTables({ + query, + connectionType: foundConnection.type as ConnectionTypesEnum, + connectionId: foundConnection.id, + validateTableRead: (referencedTableName) => + this.cedarPermissions.improvedCheckTableRead(userId, foundConnection.id, referencedTableName), + listAllTableNames: async () => (await dataAccessObject.getTablesFromDB()).map((table) => table.tableName), + }); + + const wrappedQuery = wrapQueryWithLimit(query, foundConnection.type as ConnectionTypesEnum); + const queryResult = await dataAccessObject.executeRawQuery(wrappedQuery, tableName, userEmail); + + return { result: queryResult }; + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/get-ai-connection-context.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/get-ai-connection-context.use.case.ts new file mode 100644 index 000000000..c95390f43 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/get-ai-connection-context.use.case.ts @@ -0,0 +1,40 @@ +import { Inject, Injectable, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { AiDataRequestDs } from '../data-structures/agents.ds.js'; +import { AiConnectionContextRO } from '../data-structures/agents-responses.ds.js'; +import { setupAiConnection } from '../utils/ai-data-access.helpers.js'; +import { IGetAiConnectionContext } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class GetAiConnectionContextUseCase + extends AbstractUseCase + implements IGetAiConnectionContext +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: AiDataRequestDs): Promise { + const { connectionId, userId, masterPassword } = inputData; + + const { foundConnection, isMongoDb, userEmail } = await setupAiConnection( + this._dbContext, + connectionId, + masterPassword, + userId, + ); + + return { + connectionId: foundConnection.id, + type: foundConnection.type as string, + schema: foundConnection.schema ?? null, + isMongoDb, + userEmail, + }; + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/get-ai-table-structure.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/get-ai-table-structure.use.case.ts new file mode 100644 index 000000000..9bdaaf850 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/get-ai-table-structure.use.case.ts @@ -0,0 +1,44 @@ +import { Inject, Injectable, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarPermissionsService } from '../../../entities/cedar-authorization/cedar-permissions.service.js'; +import { GetAiTableStructureDs } from '../data-structures/agents.ds.js'; +import { assertUserCanReadTables, getTableStructureInfo, setupAiConnection } from '../utils/ai-data-access.helpers.js'; +import { IGetAiTableStructure } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class GetAiTableStructureUseCase + extends AbstractUseCase> + implements IGetAiTableStructure +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarPermissions: CedarPermissionsService, + ) { + super(); + } + + protected async implementation(inputData: GetAiTableStructureDs): Promise> { + const { connectionId, userId, masterPassword, tableName } = inputData; + + const { foundConnection, dataAccessObject, userEmail } = await setupAiConnection( + this._dbContext, + connectionId, + masterPassword, + userId, + ); + + await assertUserCanReadTables(this.cedarPermissions, [tableName], userId, foundConnection.id); + + return await getTableStructureInfo( + this.cedarPermissions, + dataAccessObject, + tableName, + userEmail, + foundConnection, + userId, + ); + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/scan-and-create-settings.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/scan-and-create-settings.use.case.ts new file mode 100644 index 000000000..5ca494a63 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/scan-and-create-settings.use.case.ts @@ -0,0 +1,74 @@ +import { HttpStatus, Inject, Injectable, Scope } from '@nestjs/common'; +import Sentry from '@sentry/minimal'; +import { Response } from 'express'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { SharedJobsService } from '../../../entities/shared-jobs/shared-jobs.service.js'; +import { ConnectionNotFoundException } from '../../../exceptions/custom-exceptions/connection-not-found-exception.js'; +import { getErrorMessage } from '../../../helpers/get-error-message.js'; +import { ScanAndCreateSettingsDs } from '../data-structures/agents.ds.js'; +import { IScanAndCreateSettings } from './agents-use-cases.interface.js'; + +// Streaming variant of the AI settings/widgets scan for the agents +// microservice. Mirrors `RequestAISettingsAndWidgetsCreationUseCase`: the scan +// itself (table discovery, AI generation, validation, persistence) is shared +// with the connection-creation flow via `SharedJobsService` and stays in this +// backend; progress chunks are streamed so the agents service can pipe them +// through to the client unchanged. +@Injectable({ scope: Scope.REQUEST }) +export class ScanAndCreateSettingsUseCase + extends AbstractUseCase + implements IScanAndCreateSettings +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly sharedJobsService: SharedJobsService, + ) { + super(); + } + + protected async implementation(inputData: ScanAndCreateSettingsDs): Promise { + const { connectionId, masterPassword, response } = inputData; + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + connectionId, + masterPassword as string, + ); + if (!connection) { + throw new ConnectionNotFoundException(HttpStatus.BAD_REQUEST); + } + + this.setupResponseHeaders(response); + + try { + await this.sharedJobsService.scanDatabaseAndCreateSettingsAndWidgetsWithAI(connection, (chunk) => + this.writeChunk(response, chunk), + ); + this.writeChunk(response, { type: 'complete' }); + response.end(); + } catch (error) { + Sentry.captureException(error); + if (!response.headersSent) { + response.status(500).send({ error: 'An error occurred while processing your request.' }); + return; + } + this.writeChunk(response, { type: 'error', message: getErrorMessage(error) }); + response.end(); + } + } + + private setupResponseHeaders(response: Response): void { + response.setHeader('Content-Type', 'text/event-stream'); + response.setHeader('Cache-Control', 'no-cache'); + response.setHeader('Connection', 'keep-alive'); + } + + private writeChunk( + response: Response, + chunk: { type: 'message'; text: string } | { type: 'complete' } | { type: 'error'; message: string }, + ): void { + response.write(JSON.stringify(chunk) + '\n'); + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/validate-connection-edit.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/validate-connection-edit.use.case.ts new file mode 100644 index 000000000..c93a6194f --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/validate-connection-edit.use.case.ts @@ -0,0 +1,40 @@ +import { ForbiddenException, Inject, Injectable, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarAction } from '../../../entities/cedar-authorization/cedar-action-map.js'; +import { CedarAuthorizationService } from '../../../entities/cedar-authorization/cedar-authorization.service.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { ValidateConnectionEditDs } from '../data-structures/agents.ds.js'; +import { PermissionAllowedRO } from '../data-structures/agents-responses.ds.js'; +import { IValidateConnectionEdit } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ValidateConnectionEditUseCase + extends AbstractUseCase + implements IValidateConnectionEdit +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarAuthService: CedarAuthorizationService, + ) { + super(); + } + + protected async implementation(inputData: ValidateConnectionEditDs): Promise { + const { userId, connectionId } = inputData; + + const allowed = await this.cedarAuthService.validate({ + userId, + action: CedarAction.ConnectionEdit, + connectionId, + }); + + if (!allowed) { + throw new ForbiddenException(Messages.DONT_HAVE_PERMISSIONS); + } + + return { allowed: true }; + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/validate-table-ai-request.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/validate-table-ai-request.use.case.ts new file mode 100644 index 000000000..36e77df99 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/validate-table-ai-request.use.case.ts @@ -0,0 +1,41 @@ +import { ForbiddenException, Inject, Injectable, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarAction } from '../../../entities/cedar-authorization/cedar-action-map.js'; +import { CedarAuthorizationService } from '../../../entities/cedar-authorization/cedar-authorization.service.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { ValidateTableAiRequestDs } from '../data-structures/agents.ds.js'; +import { PermissionAllowedRO } from '../data-structures/agents-responses.ds.js'; +import { IValidateTableAiRequest } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ValidateTableAiRequestUseCase + extends AbstractUseCase + implements IValidateTableAiRequest +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarAuthService: CedarAuthorizationService, + ) { + super(); + } + + protected async implementation(inputData: ValidateTableAiRequestDs): Promise { + const { userId, connectionId, tableName } = inputData; + + const allowed = await this.cedarAuthService.validate({ + userId, + action: CedarAction.TableAiRequest, + connectionId, + tableName, + }); + + if (!allowed) { + throw new ForbiddenException(Messages.DONT_HAVE_PERMISSIONS); + } + + return { allowed: true }; + } +} diff --git a/backend/src/microservices/agents-microservice/use-cases/validate-user-token.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/validate-user-token.use.case.ts new file mode 100644 index 000000000..95f457c4d --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/validate-user-token.use.case.ts @@ -0,0 +1,78 @@ +import { HttpException, Inject, Injectable, Scope, UnauthorizedException } from '@nestjs/common'; +import Sentry from '@sentry/minimal'; +import jwt from 'jsonwebtoken'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { JwtScopesEnum } from '../../../entities/user/enums/jwt-scopes.enum.js'; +import { TwoFaRequiredException } from '../../../exceptions/custom-exceptions/two-fa-required-exception.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { appConfig } from '../../../shared/config/app-config.js'; +import { ValidatedUserTokenRO } from '../data-structures/agents-responses.ds.js'; +import { IValidateUserToken } from './agents-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ValidateUserTokenUseCase + extends AbstractUseCase + implements IValidateUserToken +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(token: string): Promise { + if (!token) { + throw new UnauthorizedException('Token is missing'); + } + + const isLoggedOut = await this._dbContext.logOutRepository.isLoggedOut(token); + if (isLoggedOut) { + throw new UnauthorizedException('Token is invalid'); + } + + try { + const jwtSecret = appConfig.auth.jwtSecret; + if (!jwtSecret) { + throw new UnauthorizedException('JWT verification failed'); + } + const data = jwt.verify(token, jwtSecret) as jwt.JwtPayload; + const userId = data.id; + + if (!userId) { + throw new UnauthorizedException('JWT verification failed'); + } + + const foundUser = await this._dbContext.userRepository.findOneUserById(userId); + if (!foundUser) { + throw new UnauthorizedException('JWT verification failed'); + } + + if (foundUser.suspended) { + throw new UnauthorizedException(Messages.ACCOUNT_SUSPENDED); + } + + const addedScope: Array = data.scope; + if (addedScope && addedScope.length > 0) { + if (addedScope.includes(JwtScopesEnum.TWO_FA_ENABLE)) { + throw new TwoFaRequiredException(); + } + } + + return { + sub: userId, + email: data.email ?? null, + exp: data.exp ?? null, + iat: data.iat ?? null, + }; + } catch (e) { + Sentry.captureException(e); + if (e instanceof HttpException) { + throw e; + } + throw new UnauthorizedException(Messages.AUTHORIZATION_REJECTED); + } + } +} diff --git a/backend/src/microservices/agents-microservice/utils/ai-data-access.helpers.ts b/backend/src/microservices/agents-microservice/utils/ai-data-access.helpers.ts new file mode 100644 index 000000000..bb623b0bd --- /dev/null +++ b/backend/src/microservices/agents-microservice/utils/ai-data-access.helpers.ts @@ -0,0 +1,153 @@ +import { BadRequestException, ForbiddenException, HttpStatus, Logger } from '@nestjs/common'; +import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { IDataAccessObject } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object.interface.js'; +import { IDataAccessObjectAgent } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object-agent.interface.js'; +import { collectMongoPipelineCollections } from '../../../ai-core/tools/collect-mongo-pipeline-collections.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { CedarPermissionsService } from '../../../entities/cedar-authorization/cedar-permissions.service.js'; +import { ConnectionEntity } from '../../../entities/connection/connection.entity.js'; +import { ConnectionNotFoundException } from '../../../exceptions/custom-exceptions/connection-not-found-exception.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { isConnectionTypeAgent } from '../../../helpers/is-connection-entity-agent.js'; + +const logger = new Logger('AgentsAiDataAccess'); + +export type AiConnectionSetup = { + foundConnection: ConnectionEntity; + dataAccessObject: IDataAccessObject | IDataAccessObjectAgent; + isMongoDb: boolean; + userEmail: string; +}; + +export async function setupAiConnection( + dbContext: IGlobalDatabaseContext, + connectionId: string, + masterPassword: string | null, + userId: string, +): Promise { + const foundConnection = await dbContext.connectionRepository.findAndDecryptConnection( + connectionId, + masterPassword as string, + ); + + if (!foundConnection) { + throw new ConnectionNotFoundException(HttpStatus.NOT_FOUND); + } + + let userEmail = ''; + if (isConnectionTypeAgent(foundConnection.type)) { + userEmail = (await dbContext.userRepository.getUserEmailOrReturnNull(userId)) ?? ''; + } + + const connectionProperties = await dbContext.connectionPropertiesRepository.findConnectionProperties(connectionId); + + if (connectionProperties && !connectionProperties.allow_ai_requests) { + throw new BadRequestException(Messages.AI_REQUESTS_NOT_ALLOWED); + } + + const dataAccessObject = getDataAccessObject(foundConnection); + const databaseType = foundConnection.type; + const isMongoDb = databaseType === ConnectionTypesEnum.mongodb || databaseType === ConnectionTypesEnum.agent_mongodb; + + return { foundConnection, dataAccessObject, isMongoDb, userEmail }; +} + +export async function assertUserCanReadTables( + cedarPermissions: CedarPermissionsService, + tableNames: Array, + userId: string, + connectionId: string, +): Promise { + const uniqueTableNames = Array.from( + new Set(tableNames.map((name) => name?.trim()).filter((name): name is string => Boolean(name))), + ); + + for (const tableName of uniqueTableNames) { + const canRead = await cedarPermissions.improvedCheckTableRead(userId, connectionId, tableName); + if (!canRead) { + logger.warn( + `AI request blocked for user ${userId} on connection ${connectionId}: ` + + `no read permission for table "${tableName}"`, + ); + throw new ForbiddenException(Messages.NO_READ_PERMISSION_FOR_TABLE(tableName)); + } + } +} + +export async function assertUserCanReadPipelineCollections( + cedarPermissions: CedarPermissionsService, + pipeline: string, + baseCollection: string, + userId: string, + connectionId: string, + dataAccessObject: IDataAccessObject | IDataAccessObjectAgent, +): Promise { + const collected = collectMongoPipelineCollections(pipeline); + + let collectionsToCheck: Array; + if (collected.kind === 'tables') { + collectionsToCheck = [baseCollection, ...collected.tables]; + } else { + logger.warn( + `AI pipeline permission check could not resolve referenced collections for connection ${connectionId} ` + + `(reason: ${collected.reason}); falling back to all-collections read check.`, + ); + collectionsToCheck = (await dataAccessObject.getTablesFromDB()).map((table) => table.tableName); + } + + await assertUserCanReadTables(cedarPermissions, collectionsToCheck, userId, connectionId); +} + +export async function getTableStructureInfo( + cedarPermissions: CedarPermissionsService, + dao: IDataAccessObject | IDataAccessObjectAgent, + tableName: string, + userEmail: string, + foundConnection: ConnectionEntity, + userId: string, +): Promise> { + const [tableStructure, tableForeignKeys, referencedTableNamesAndColumns] = await Promise.all([ + dao.getTableStructure(tableName, userEmail), + dao.getTableForeignKeys(tableName, userEmail), + dao.getReferencedTableNamesAndColumns(tableName, userEmail), + ]); + + const referencedTablesStructures = []; + const structurePromises = referencedTableNamesAndColumns.flatMap((referencedTable) => + referencedTable.referenced_by.map(async (table) => { + const canRead = await cedarPermissions.improvedCheckTableRead(userId, foundConnection.id, table.table_name); + if (!canRead) { + return null; + } + const structure = await dao.getTableStructure(table.table_name, userEmail); + return { tableName: table.table_name, structure }; + }), + ); + referencedTablesStructures.push(...(await Promise.all(structurePromises)).filter((item) => item !== null)); + + const foreignTablesStructures = []; + const foreignTablesStructurePromises = tableForeignKeys.map(async (foreignKey) => { + const canRead = await cedarPermissions.improvedCheckTableRead( + userId, + foundConnection.id, + foreignKey.referenced_table_name, + ); + if (!canRead) { + return null; + } + const structure = await dao.getTableStructure(foreignKey.referenced_table_name, userEmail); + return { tableName: foreignKey.referenced_table_name, structure }; + }); + foreignTablesStructures.push(...(await Promise.all(foreignTablesStructurePromises)).filter((item) => item !== null)); + + return { + tableStructure, + tableName, + schema: foundConnection.schema || null, + tableForeignKeys, + referencedTableNamesAndColumns, + referencedTablesStructures, + foreignTablesStructures, + }; +}