Skip to content

Commit 755d3d9

Browse files
committed
streaming from dmm enabled
1 parent 34ec20c commit 755d3d9

2 files changed

Lines changed: 20 additions & 4 deletions

File tree

Source-Connector/api/services/service.js

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ const path = require("path");
1818
let attrWithUrl = config.orion?.attrWithUrl || "datasetUrl";
1919
require("../../inputConnectors/apiConnector");
2020

21-
async function insertResponseInDB(size) {
21+
async function insertResponseInDB(size, urlValue) {
2222
let stats;
2323
let sizeRead = 0;
2424
logger.info(`Attesa del file di stream di dimensione minima: ${size} bytes`);
@@ -198,8 +198,24 @@ module.exports = {
198198
retry -= 2;
199199
try {
200200
logger.info("Inserting datapoints into DB...");
201-
logger.info(response.data[0], " MB");
202-
await insertResponseInDB(response.data[0]); //.map(d => {return {...d, dimensions : {...(d.dimensions), year : d.dimensions.time}}})) //TODO check if datapoints or other data and generalize insertion
201+
logger.info(response.data);
202+
let outputId = response.data.id
203+
let lastId
204+
let purged = false
205+
for (let chunkIndex = 0; (response.data[0] || response.data.id); chunkIndex++) {
206+
//while (response.data[0] || response.data.id) {
207+
logger.info(response.data.status)
208+
response = await axios.get((config.sessionEdnpoint || "http://localhost:5500/api/session?") + "id=" + outputId + "&lastId=" + lastId + "&index=" + chunkIndex, {
209+
headers: {
210+
Authorization: `Bearer ${bearerToken}`
211+
}
212+
})
213+
if (!purged)
214+
await Datapoints.deleteMany({survey: response.data[0].survey});
215+
await Datapoints.insertMany(response.data); //.map(d => {return {...d, dimensions : {...(d.dimensions), year : d.dimensions.time}}})) //TODO check if datapoints or other data and generalize insertion
216+
lastId = response.data[response.data.length - 1]?._id
217+
purged = true;
218+
}
203219
} catch (error) {
204220
logger.error("Error inserting datapoints:", error);
205221
}

Source-Connector/utils/keycloak.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
const axios = require("axios");
22
const config = require("../config.js")
3-
const keycloakBaseUrl = config.authConfig.idmHost + "/realms/smartera";
3+
const keycloakBaseUrl = config.authConfig.idmHost + "/realms/" + (config.authConfig.authRealm || "master")
44
const { clientId, username, password } = config.authConfig;
55
const fs = require('fs');
66
const path = './token.js';

0 commit comments

Comments
 (0)