diff --git a/backend/actions/Model/aggregate.js b/backend/actions/Model/aggregate.js new file mode 100644 index 00000000..c7b23b64 --- /dev/null +++ b/backend/actions/Model/aggregate.js @@ -0,0 +1,62 @@ +'use strict'; + +const Archetype = require('archetype'); +const { EJSON } = require('mongoose').mongo.BSON; +const authorize = require('../../authorize'); + +const AggregateParams = new Archetype({ + model: { + $type: 'string', + $required: true + }, + pipeline: { + $type: Archetype.Any, + $required: true + }, + limit: { + $type: 'number', + $default: 20 + }, + roles: { + $type: ['string'] + } +}).compile('AggregateParams'); + +module.exports = ({ db }) => async function aggregate(params) { + params = new AggregateParams(params); + const { model, roles } = params; + await authorize('Model.aggregate', roles); + + const Model = db.models[model]; + if (Model == null) { + throw new Error(`Model ${model} not found`); + } + + if (!Array.isArray(params.pipeline)) { + throw new Error('`pipeline` must be an array'); + } + + let pipeline; + try { + pipeline = EJSON.deserialize(params.pipeline); + } catch (err) { + throw new Error(`Invalid pipeline (EJSON): ${err.message}`); + } + + if (!Array.isArray(pipeline)) { + throw new Error('`pipeline` must be an array'); + } + + pipeline = pipeline.map((stage, index) => { + if (stage == null || Array.isArray(stage) || typeof stage !== 'object') { + throw new Error(`Invalid stage at index ${index}`); + } + return stage; + }); + + const limit = Math.max(1, Math.min(200, Math.floor(params.limit ?? 20))); + pipeline.push({ $limit: limit }); + + const docs = await Model.aggregate(pipeline).exec(); + return { docs }; +}; diff --git a/backend/actions/Model/index.js b/backend/actions/Model/index.js index aacf041c..aee9afc8 100644 --- a/backend/actions/Model/index.js +++ b/backend/actions/Model/index.js @@ -1,6 +1,7 @@ 'use strict'; exports.addField = require('./addField'); +exports.aggregate = require('./aggregate'); exports.createChatMessage = require('./createChatMessage'); exports.createDocument = require('./createDocument'); exports.deleteDocument = require('./deleteDocument'); diff --git a/backend/authorize.js b/backend/authorize.js index a4e0a9bd..a519e36d 100644 --- a/backend/authorize.js +++ b/backend/authorize.js @@ -17,6 +17,7 @@ const actionsToRequiredRoles = { 'Model.deleteDocument': ['owner', 'admin', 'member'], 'Model.deleteDocuments': ['owner', 'admin', 'member'], 'Model.dropIndex': ['owner', 'admin'], + 'Model.aggregate': ['owner', 'admin', 'member', 'readonly'], 'Model.executeDocumentScript': ['owner', 'admin', 'member'], 'Model.exportQueryResults': ['owner', 'admin', 'member', 'readonly'], 'Model.getDocument': ['owner', 'admin', 'member', 'readonly'], diff --git a/frontend/src/aggregation-builder/aggregation-builder.html b/frontend/src/aggregation-builder/aggregation-builder.html new file mode 100644 index 00000000..10d5402e --- /dev/null +++ b/frontend/src/aggregation-builder/aggregation-builder.html @@ -0,0 +1,225 @@ +
+
+
+
+
+
+

Aggregation options

+

Model, how many documents to return, and pipeline actions.

+
+
+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+
+ +
+
+

Pipeline stages

+
+
+ +
+
+ Stage {{index + 1}} + +
+
+
+
+ Operator + +
+ + +

+ Same idea as the create document editor: JSON or JavaScript-style objects with + ObjectId(), + new Date(), + RegExp, etc. Types are sent with EJSON so they match on the server. +

+
+ +
+
+ Pipeline JSON (EJSON) + · through stage {{index + 1}} +
+ 
{{ pipelinePreviewThrough(index) }}
+
+ +
+
Stage output preview
+
+
+ + +
+
+ +
+
+
+
+
+
+
+
+ +
+
+ Results + Running… +
+
+

{{errorMessage}}

+

+ Click Run Now to execute the aggregation and show results here. +

+

Fetching results…

+
+

+ Showing {{ visibleResults.length }} of {{ results.length }} document(s). +

+
+ +
+ +
+
+
+
+
diff --git a/frontend/src/aggregation-builder/aggregation-builder.js b/frontend/src/aggregation-builder/aggregation-builder.js new file mode 100644 index 00000000..b56d55e2 --- /dev/null +++ b/frontend/src/aggregation-builder/aggregation-builder.js @@ -0,0 +1,319 @@ +'use strict'; + +const api = require('../api'); +const template = require('./aggregation-builder.html'); +const { BSON, EJSON } = require('mongodb/lib/bson'); + +const ObjectId = new Proxy(BSON.ObjectId, { + apply(target, thisArg, argumentsList) { + return new target(...argumentsList); + } +}); + +/** + * Serialize a pipeline for the API — same EJSON round-trip as Model.createDocument. + */ +function serializePipelineForWire(pipeline) { + return EJSON.serialize(pipeline); +} + +/** + * Parse a stage body as JSON, or as a JavaScript object literal like the + * create-document modal (unquoted keys, ObjectId(), new Date(), RegExp, etc.) + * via a controlled Function scope. + */ +function parseStageBody(text) { + const trimmed = typeof text === 'string' ? text.trim() : ''; + if (!trimmed) { + return {}; + } + try { + return JSON.parse(trimmed); + } catch { + // Not strict JSON — try a JS expression in a controlled scope. + } + try { + const fn = new Function( + 'ObjectId', + 'Date', + 'Math', + 'RegExp', + `return (${trimmed});` + ); + return fn(ObjectId, Date, Math, RegExp); + } catch (err) { + throw new Error(err.message || String(err)); + } +} + +const STAGE_OPERATORS = [ + '$match', + '$project', + '$group', + '$sort', + '$limit', + '$skip', + '$unwind', + '$lookup', + '$addFields', + '$set', + '$unset', + '$count', + '$facet' +]; +const STAGE_PREVIEW_LIMIT = 3; +const RESULT_PAGE_SIZE = 20; + +function createDefaultStage() { + return { + id: Math.random().toString(36).slice(2), + operator: '$match', + bodyText: '{}', + previewDocs: [], + previewError: '', + previewLoading: false, + previewExpanded: false, + previewLoaded: false + }; +} + +module.exports = app => app.component('aggregation-builder', { + template: template, + props: ['roles'], + data: () => ({ + models: [], + selectedModel: null, + resultLimit: 20, + stages: [createDefaultStage()], + stageOperators: STAGE_OPERATORS, + isRunning: false, + errorMessage: '', + results: [], + visibleResultsCount: RESULT_PAGE_SIZE, + resultsRenderKey: 0, + activeRunId: 0 + }), + computed: { + hasPipelineErrors() { + return this.stages.some(stage => this.getStageError(stage) != null); + }, + pipelineStageErrors() { + const errors = []; + for (let i = 0; i < this.stages.length; i++) { + const message = this.getStageError(this.stages[i]); + if (message) { + errors.push({ stageNumber: i + 1, message }); + } + } + return errors; + }, + pipelineSignature() { + return this.stages.map(stage => `${stage.operator}::${stage.bodyText || ''}`).join('||'); + }, + visibleResults() { + return this.results.slice(0, this.visibleResultsCount); + }, + hasMoreResults() { + return this.visibleResultsCount < this.results.length; + }, + nextLoadMoreCount() { + return Math.min(RESULT_PAGE_SIZE, this.results.length - this.visibleResultsCount); + }, + visibleResultsExpandedFields() { + return this.visibleResults.map((_, i) => `root[${i}]`); + } + }, + async mounted() { + const { models } = await api.Model.listModels(); + this.models = models || []; + if (this.models.length > 0) { + this.selectedModel = this.models[0]; + } + }, + methods: { + addStage() { + this.stages.push(createDefaultStage()); + this.$nextTick(() => { + const rows = this.$refs.workflowStageRows; + const el = Array.isArray(rows) ? rows[rows.length - 1] : rows; + if (el && typeof el.scrollIntoView === 'function') { + el.scrollIntoView({ behavior: 'smooth', block: 'end' }); + } + }); + }, + removeStage(index) { + if (index < 0 || index >= this.stages.length) { + return; + } + if (this.stages.length === 1) { + this.stages.splice(0, 1, createDefaultStage()); + return; + } + this.stages.splice(index, 1); + }, + getStageError(stage) { + const text = typeof stage.bodyText === 'string' ? stage.bodyText.trim() : ''; + if (!text) { + return null; + } + let parsedBody = null; + try { + parsedBody = parseStageBody(text); + } catch (err) { + return err.message; + } + if (parsedBody == null || Array.isArray(parsedBody) || typeof parsedBody !== 'object') { + return 'Stage body must be a plain object'; + } + return null; + }, + buildPipeline() { + return this.stages.map(stage => { + const text = typeof stage.bodyText === 'string' ? stage.bodyText.trim() : ''; + let parsedBody = {}; + if (text) { + try { + parsedBody = parseStageBody(text); + } catch (err) { + parsedBody = {}; + } + } + return { [stage.operator]: parsedBody }; + }); + }, + pipelinePreviewThrough(index) { + const slice = this.buildPipeline().slice(0, index + 1); + try { + return JSON.stringify(serializePipelineForWire(slice), null, 2); + } catch (err) { + return `/* Could not serialize pipeline for preview: ${err.message} */\n${JSON.stringify(slice, null, 2)}`; + } + }, + formatDoc(doc) { + return JSON.stringify(doc, null, 2); + }, + loadMoreResults() { + this.visibleResultsCount = Math.min(this.visibleResultsCount + RESULT_PAGE_SIZE, this.results.length); + }, + toggleStagePreview(stage) { + stage.previewExpanded = !stage.previewExpanded; + }, + pipelineThroughIndexHasErrors(index) { + for (let i = 0; i <= index; i++) { + if (this.getStageError(this.stages[i]) != null) { + return true; + } + } + return false; + }, + runStagePreview(index) { + if (index < 0 || index >= this.stages.length || !this.selectedModel) { + return; + } + if (this.pipelineThroughIndexHasErrors(index)) { + return; + } + const stage = this.stages[index]; + stage.previewExpanded = true; + this.loadSingleStagePreview(index); + }, + async loadSingleStagePreview(index) { + if (index < 0 || index >= this.stages.length) { + return; + } + const stage = this.stages[index]; + if (!this.selectedModel) { + return; + } + const token = (stage._previewRequestId = (stage._previewRequestId || 0) + 1); + stage.previewLoading = true; + stage.previewError = ''; + try { + const partialPipeline = this.buildPipeline().slice(0, index + 1); + const wirePipeline = serializePipelineForWire(partialPipeline); + const { docs } = await api.Model.aggregate({ + model: this.selectedModel, + pipeline: wirePipeline, + limit: STAGE_PREVIEW_LIMIT, + roles: this.roles + }); + if (token !== stage._previewRequestId) { + return; + } + stage.previewDocs = docs || []; + stage.previewLoaded = true; + } catch (err) { + if (token !== stage._previewRequestId) { + return; + } + stage.previewError = err?.response?.data?.message || err.message || 'Could not preview this stage'; + stage.previewDocs = []; + stage.previewLoaded = true; + } finally { + if (token === stage._previewRequestId) { + stage.previewLoading = false; + } + } + }, + invalidateStagePreviews() { + for (const stage of this.stages) { + stage._previewRequestId = (stage._previewRequestId || 0) + 1; + stage.previewDocs = []; + stage.previewError = ''; + stage.previewLoading = false; + stage.previewLoaded = false; + } + }, + async runAggregation() { + this.errorMessage = ''; + const pipeline = this.buildPipeline(); + if (this.hasPipelineErrors) { + this.errorMessage = 'Fix invalid stage syntax before running.'; + return; + } + if (!this.selectedModel) { + return; + } + const runId = ++this.activeRunId; + this.isRunning = true; + try { + let wirePipeline; + try { + wirePipeline = serializePipelineForWire(pipeline); + } catch (err) { + this.errorMessage = `Could not serialize pipeline: ${err.message}`; + return; + } + const { docs } = await api.Model.aggregate({ + model: this.selectedModel, + pipeline: wirePipeline, + limit: this.resultLimit, + roles: this.roles + }); + if (runId !== this.activeRunId) { + return; + } + this.results = docs || []; + this.visibleResultsCount = RESULT_PAGE_SIZE; + this.resultsRenderKey += 1; + } catch (err) { + if (runId !== this.activeRunId) { + return; + } + this.errorMessage = err?.response?.data?.message || err.message || 'Aggregation failed'; + } finally { + if (runId === this.activeRunId) { + this.isRunning = false; + } + } + } + }, + watch: { + pipelineSignature() { + this.invalidateStagePreviews(); + }, + selectedModel() { + this.invalidateStagePreviews(); + } + } +}); diff --git a/frontend/src/api.js b/frontend/src/api.js index 3dfb4478..2d404f09 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -176,6 +176,9 @@ if (window.MONGOOSE_STUDIO_CONFIG.isLambda) { }, updateDocuments: function updateDocuments(params) { return client.post('', { action: 'Model.updateDocuments', ...params }).then(res => res.data); + }, + aggregate(params) { + return client.post('', { action: 'Model.aggregate', ...params }).then(res => res.data); } }; exports.Task = { @@ -519,6 +522,9 @@ if (window.MONGOOSE_STUDIO_CONFIG.isLambda) { }, updateDocuments: function updateDocument(params) { return client.post('/Model/updateDocuments', params).then(res => res.data); + }, + aggregate(params) { + return client.post('/Model/aggregate', params).then(res => res.data); } }; exports.Task = { diff --git a/frontend/src/navbar/navbar.html b/frontend/src/navbar/navbar.html index 8be86af2..ec6d7c2b 100644 --- a/frontend/src/navbar/navbar.html +++ b/frontend/src/navbar/navbar.html @@ -41,6 +41,16 @@ + Aggregation + + Aggregation + + + Aggregation + + Aggregation + + app.component('navbar', { chatView() { return ['chat index', 'chat'].includes(this.$route.name); }, + aggregationBuilderView() { + return this.$route.name === 'aggregationBuilder'; + }, taskView() { return ['tasks', 'taskByName', 'taskSingle'].includes(this.$route.name); }, diff --git a/frontend/src/routes.js b/frontend/src/routes.js index 5b15f7f4..eddfbdb7 100644 --- a/frontend/src/routes.js +++ b/frontend/src/routes.js @@ -2,14 +2,14 @@ // Role-based access control configuration const roleAccess = { - owner: ['root', 'model', 'document', 'dashboards', 'dashboard', 'team', 'chat', 'tasks', 'taskByName', 'taskSingle'], - admin: ['root', 'model', 'document', 'dashboards', 'dashboard', 'team', 'chat', 'tasks', 'taskByName', 'taskSingle'], - member: ['root', 'model', 'document', 'dashboards', 'dashboard', 'chat', 'tasks', 'taskByName', 'taskSingle'], - readonly: ['root', 'model', 'document', 'chat'], + owner: ['root', 'model', 'document', 'dashboards', 'dashboard', 'team', 'chat', 'aggregationBuilder', 'tasks', 'taskByName', 'taskSingle'], + admin: ['root', 'model', 'document', 'dashboards', 'dashboard', 'team', 'chat', 'aggregationBuilder', 'tasks', 'taskByName', 'taskSingle'], + member: ['root', 'model', 'document', 'dashboards', 'dashboard', 'chat', 'aggregationBuilder', 'tasks', 'taskByName', 'taskSingle'], + readonly: ['root', 'model', 'document', 'chat', 'aggregationBuilder'], dashboards: ['dashboards', 'dashboard'] }; -const allowedRoutesForLocalDev = ['document', 'dashboards', 'dashboard', 'root', 'chat', 'model', 'tasks', 'taskByName', 'taskSingle']; +const allowedRoutesForLocalDev = ['document', 'dashboards', 'dashboard', 'root', 'chat', 'aggregationBuilder', 'model', 'tasks', 'taskByName', 'taskSingle']; // Helper function to check if a role has access to a route function hasAccess(roles, routeName) { @@ -107,6 +107,14 @@ module.exports = { meta: { authorized: true } + }, + { + path: '/aggregation-builder', + name: 'aggregationBuilder', + component: 'aggregation-builder', + meta: { + authorized: true + } } ], roleAccess, diff --git a/test/Model.aggregate.test.js b/test/Model.aggregate.test.js new file mode 100644 index 00000000..4c92291a --- /dev/null +++ b/test/Model.aggregate.test.js @@ -0,0 +1,134 @@ +'use strict'; + +const assert = require('assert'); +const mongoose = require('mongoose'); +const { actions, connection } = require('./setup.test'); + +describe('Model.aggregate()', function () { + const AggregateTest = connection.model('AggregateTest', new mongoose.Schema({ + name: String, + n: Number + })); + + afterEach(async function () { + await AggregateTest.deleteMany(); + }); + + it('returns documents from a valid pipeline', async function () { + await AggregateTest.create([{ name: 'a', n: 1 }, { name: 'b', n: 2 }]); + + const res = await actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: { name: 'a' } }], + roles: ['readonly'] + }); + + assert.ok(res.docs); + assert.strictEqual(res.docs.length, 1); + assert.strictEqual(res.docs[0].name, 'a'); + }); + + it('appends a server-side $limit and does not return more than the limit', async function () { + await AggregateTest.create([{ n: 1 }, { n: 2 }, { n: 3 }, { n: 4 }, { n: 5 }]); + + const res = await actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }, { $sort: { n: 1 } }], + limit: 2, + roles: ['member'] + }); + + assert.strictEqual(res.docs.length, 2); + assert.strictEqual(res.docs[0].n, 1); + assert.strictEqual(res.docs[1].n, 2); + }); + + it('defaults limit to 20 when omitted', async function () { + const docsToCreate = Array.from({ length: 25 }, (_, i) => ({ n: i })); + await AggregateTest.create(docsToCreate); + + const res = await actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }], + roles: ['admin'] + }); + + assert.strictEqual(res.docs.length, 20); + }); + + it('clamps limit to 200', async function () { + const docsToCreate = Array.from({ length: 205 }, (_, i) => ({ n: i })); + await AggregateTest.create(docsToCreate); + + const res = await actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }], + limit: 500, + roles: ['admin'] + }); + + assert.strictEqual(res.docs.length, 200); + }); + + it('clamps a non-positive limit to 1', async function () { + await AggregateTest.create([{ n: 1 }, { n: 2 }]); + + const res = await actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }, { $sort: { n: 1 } }], + limit: 0, + roles: ['admin'] + }); + + assert.strictEqual(res.docs.length, 1); + assert.strictEqual(res.docs[0].n, 1); + }); + + it('throws if the model does not exist', async function () { + await assert.rejects( + () => + actions.Model.aggregate({ + model: 'DoesNotExistModel', + pipeline: [{ $match: {} }], + roles: ['admin'] + }), + /Model DoesNotExistModel not found/ + ); + }); + + it('throws if pipeline is not an array', async function () { + await assert.rejects( + () => + actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: { $match: {} }, + roles: ['admin'] + }), + /`pipeline` must be an array/ + ); + }); + + it('throws if a stage is not a non-null object', async function () { + await assert.rejects( + () => + actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }, null], + roles: ['admin'] + }), + /Invalid stage at index 1/ + ); + }); + + it('throws if the caller is not authorized', async function () { + await assert.rejects( + () => + actions.Model.aggregate({ + model: 'AggregateTest', + pipeline: [{ $match: {} }], + roles: ['nope'] + }), + /Unauthorized to take action Model\.aggregate/ + ); + }); +});