Skip to content

Commit 372bb4b

Browse files
PercoccoPercocco
committed
toarray operator added
1 parent 415d0dd commit 372bb4b

2 files changed

Lines changed: 66 additions & 99 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,4 @@ frontend/src/app/pages/data-model-mapper/testModule.ts
6262
backend/src/writers/orionWriter.old.js
6363
backend/output/
6464
frontend/src/app/pages/data-model-mapper/stepper_test.html
65+
*.sandbox

‎backend/src/mapHandler.js‎

Lines changed: 65 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ const utils = require('./utils/utils.js');
2424
const validator = require('./schemaHandler.js');
2525
const unorm = require('unorm');
2626
const staticPattern = /static:(.*)/;
27+
const toArrayPattern = /toarray:(.*)/;
28+
const forEachPattern = /foreach:(.*)/;
2729
const dotPattern = /(.*)\.(.*)/;
2830

2931
const log = require('./utils/logger')//.app(module);
@@ -108,10 +110,19 @@ const encodingHandler = (mapSourceSubField, source) => {
108110
*/
109111

110112
const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) => {
111-
113+
logger.debug({ parsedSourceKey, normSourceKey, schemaDestKey, source })
112114
for (let key in normSourceKey) {
115+
logger.debug({ key })
116+
117+
let schemaDestSubKey
118+
if (schemaDestKey.properties && !schemaDestKey.oneOf)
119+
schemaDestSubKey = schemaDestKey.properties[key];
120+
if (schemaDestKey.oneOf)
121+
for (let oneOfElement of schemaDestKey.oneOf)
122+
if (oneOfElement.properties && oneOfElement.properties[key])
123+
schemaDestSubKey = oneOfElement.properties[key];
124+
logger.debug({ schemaDestSubKey })
113125

114-
let schemaDestSubKey = schemaDestKey.properties[key];
115126
if (schemaDestSubKey) {
116127

117128
let schemaFieldType = schemaDestSubKey.type;
@@ -127,7 +138,7 @@ const objectHandler = (parsedSourceKey, normSourceKey, schemaDestKey, source) =>
127138
parsedSourceKey[key] = new Function("input", "return new Date(input['" + mapSourceSubField + "']).toISOString();");
128139
else if (schemaFieldType === 'string' && Array.isArray(mapSourceSubField))
129140
parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsArray(mapSourceSubField).result);
130-
else if (schemaFieldType === 'array' && Array.isArray(mapSourceSubField))
141+
else if (schemaFieldType === 'array')
131142
parsedSourceKey[key] = new Function("input", "return " + handleSourceFieldsToDestArray(mapSourceSubField));
132143
else if (schemaFieldType === 'string' && typeof mapSourceSubField === 'string' && (mapSourceSubField.startsWith("static:") || mapSourceSubField == "")) {
133144
if (mapSourceSubField == "") mapSourceSubField = "static:"
@@ -171,111 +182,42 @@ const extractFromNestedField = (source, field) => {
171182
*/
172183
const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service, group, entityIdField, NGSI_entity, minioObj, config, res) => {
173184

185+
logger.debug({ rowNumber, source, map })
174186
var result = {};
175187
// If the destKey is entityIdField and has only "static:" fields, the pair value indicates only an ID prefix
176188
// The resulting string will be concatenated with rowNumber
177189
var isIdPrefix = false;
178-
179190
for (var mapDestKey in map) {
180-
181-
let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair
191+
let mapSourceKey = map[mapDestKey]; // sourceField map object or key-value pair
182192
let singleResult = undefined;
183193
let schemaDestKey = modelSchema.allOf[0].properties[mapDestKey];
184-
185-
//// If the map key has a . , it means that the source key is an object
186-
//if (mapDestKey.test(dotPattern)){
187-
// var extrFields = mapDestKey.match(dotPattern);
188-
// // Check if there are other subfields for this object field
189-
// if (extrFields.length > 1) {
190-
191-
192-
// Check if destKey is present in modelSchema ?
193-
if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) {
194-
194+
if (schemaDestKey || mapDestKey === entityIdField || config.ignoreValidation) {// Check if destKey is present in modelSchema ?
195195
if (config.ignoreValidation && source[map[mapDestKey]])
196196
modelSchema.allOf[0].properties[mapDestKey] = { "type": typeof source[map[mapDestKey]] }
197-
198-
// If the value of key-value maping pair is a function definition, eval it.
199-
//if ( (typeof mapSourceField == "string") && mapSourceField.startsWith("function")) {
200-
// map[destKey] = utils.parseFunction(mapSourceField);
201-
202-
// Convert the single source field from map, to the final mapped single object or key-value pair, to be validated
203-
// If valid, it is added to the final result object, otherwise is discarded
204-
205-
// Normalize encoding, avoiding problems with fields name not recognized due to different source encoding
206-
var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey)));
207-
208-
// Initialize with normalized Source Key, can be replaced in the specific cases below
209-
let parsedSourceKey = normSourceKey;
210-
211-
// If the value type of mapped field is different from string, try first to extract it
212-
// If destination Schema field is OneOf
213-
if (schemaDestKey && !schemaDestKey.type && schemaDestKey.oneOf) {
214-
215-
var oneOf = schemaDestKey.oneOf;
216-
217-
/** If Destination Key is an object with coordinates it's a location type field or if it is "geometry" **/
218-
if (mapDestKey === 'location') {
219-
if (normSourceKey.type && normSourceKey.coordinates && normSourceKey.type.startsWith('static:')) {
220-
221-
var parsedStaticType = normSourceKey.type.match(staticPattern)[1];
222-
if (Array.isArray(oneOf) && oneOf.find(k => k.properties.type.enum.find(e => e == parsedStaticType))) {
223-
224-
parsedSourceKey['type'] = new Function("input", "return '" + parsedStaticType + "'");
225-
parsedSourceKey['coordinates'] = new Function("input", "return " + "[Number(input['" + normSourceKey.coordinates[0] + "']),Number(input['" + normSourceKey.coordinates[1] + "'])]");
226-
}
227-
228-
} else if (normSourceKey === 'geometry' || normSourceKey === 'location') {
229-
parsedSourceKey = new Function("input", "return input['" + normSourceKey + "'];");
230-
} else
231-
continue;
232-
}
233-
234-
/********************* Destination Key is an Object ****************************************/
235-
236-
} else if (schemaDestKey && schemaDestKey.type === 'object' && typeof normSourceKey === 'object')
237-
197+
var normSourceKey = JSON.parse(unorm.nfc(JSON.stringify(mapSourceKey)));// Normalize encoding, avoiding problems
198+
let parsedSourceKey = normSourceKey;// Initialize with normalized Source Key, can be replaced in the specific cases below
199+
logger.debug({ schemaDestKey, normSourceKey })
200+
if (schemaDestKey && schemaDestKey.type === 'object' || typeof normSourceKey === 'object')
238201
parsedSourceKey = objectHandler(parsedSourceKey, normSourceKey, schemaDestKey, source)
239-
240-
/********************* Destination Field is an Array ********************************************/
241-
242202
else if (schemaDestKey && schemaDestKey.type === 'array' && Array.isArray(normSourceKey))
243-
244203
parsedSourceKey = new Function("input", "return " + handleSourceFieldsToDestArray(normSourceKey));
245-
246-
/********************* Destination Field is a Number ********************************************/
247-
248204
else if (schemaDestKey && (schemaDestKey.type === 'number' || schemaDestKey.type === 'integer'))
249-
250205
if (Array.isArray(normSourceKey))
251206
parsedSourceKey = new Function("input", "return " + handleSourceFieldsArray(normSourceKey, 'number').result);
252-
253207
else {
254-
255208
parsedSourceKey = handleDottedField(normSourceKey);
256-
257209
if (parsedSourceKey.startsWith('[')) {
258-
259210
let num = eval('source' + parsedSourceKey);
260-
261211
if (typeof num === 'string')
262212
parsedSourceKey = new Function("input", "return Number(input['" + normSourceKey + "'])");
263-
264213
else if (typeof num === 'number')
265214
parsedSourceKey = new Function("input", "return input['" + normSourceKey + "']");
266215
}
267216
}
268-
269-
/********************* Destination Field is a String ********************************************/
270-
271217
else if (schemaDestKey && (schemaDestKey.type === 'boolean'))
272-
273218
parsedSourceKey = new Function("input", "return (input['" + normSourceKey + "'].toLowerCase() == 'true' || input['" + normSourceKey + "'] == 1 || input['" + normSourceKey + "'] == '1' ) ? true: false");
274-
275219
else if (schemaDestKey && schemaDestKey.type === 'string') {
276-
277220
if (schemaDestKey.format === 'date-time') {
278-
279221
var date = eval('source' + handleDottedField(normSourceKey));
280222
if (date === undefined || date === '')
281223
continue;
@@ -286,10 +228,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
286228
parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'");
287229
else if (typeof normSourceKey === 'string' && normSourceKey.startsWith("encode:"))
288230
parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'");
289-
290-
/********************* Destination Key is a entityId field (according to definition in config.js) **/
291231
} else if (mapDestKey == entityIdField) {
292-
293232
if (Array.isArray(normSourceKey) && normSourceKey.length !== 0) {
294233
let resIdFields = handleSourceFieldsArray(normSourceKey);
295234
parsedSourceKey = new Function("input", "return " + resIdFields.result);
@@ -299,23 +238,11 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
299238
parsedSourceKey = new Function("input", "return '" + normSourceKey.match(staticPattern)[1] + "'");
300239
else if (normSourceKey.startsWith("encode:"))
301240
parsedSourceKey = new Function("input", "return '" + encodingHandler(normSourceKey, source) + "'");
302-
303-
304241
}
305-
306-
// Add type to map field
307-
//parsedNorm.type = new Function("input", "return '" + modelSchemaDestKey.type +"'");
308-
309-
/********************* Perform actual mapping with parsed and normalized source key (parsedNorm) **/
310-
311242
if (typeof parsedSourceKey == "string")
312243
parsedSourceKey = parsedSourceKey.replaceAll('"', '')
313-
314-
//logger.debug(parsedSourceKey)
315-
//logger.debug(typeof parsedSourceKey)
316-
244+
logger.debug({ mapDestKey, parsedSourceKey, coordinates: parsedSourceKey.coordinates?.toString() })
317245
var converter = mapper.makeConverter({ [mapDestKey]: parsedSourceKey });
318-
319246
try {
320247
singleResult = converter(source);
321248
} catch (error) {
@@ -377,7 +304,7 @@ const mapObjectToDataModel = (rowNumber, source, map, modelSchema, site, service
377304
result.id = result.id.replaceAll(" ", "")
378305
} catch (error) {
379306
logger.error(error)
380-
307+
381308
logger.error("UnknownEntity")
382309
}
383310
}
@@ -482,6 +409,10 @@ const handleSourceFieldsArray = (sourceFieldArray, sourceFieldType) => {
482409
/* Map fields of the source array into a stringifed Array (source and dest are both arrays)
483410
*/
484411
const handleSourceFieldsToDestArray = (sourceFieldArray) => {
412+
//let foreachIndex = []
413+
let foreachFound = false
414+
415+
logger.debug({ sourceFieldArray })
485416

486417
if (sourceFieldArray.length > 0) {
487418
var finalArray = [];
@@ -491,11 +422,40 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
491422
sourceFieldArray.forEach(function (value, index, array) {
492423

493424
var staticMatch = value.match(staticPattern);
425+
//let toArrayMatch = value.match(toArrayPattern);
426+
let forEachMatch = value.match(forEachPattern);
494427
if (staticMatch && staticMatch.length > 0) {
495428

496429
finalArray[index] = staticMatch[1];
497430

498-
} else {
431+
}
432+
else if (forEachMatch && forEachMatch.length > 0) {
433+
if (!foreachFound) foreachFound = true
434+
//let arrayField = forEachMatch[1];
435+
//let arrayFieldCleaned = cleanValue(arrayField);
436+
//foreachIndex.push(index)
437+
let subOperator = forEachMatch[1];
438+
let forEachArgument = subOperator.split(',')[0];
439+
let forEachBody = subOperator.split(',').slice(1).join(',');
440+
if (forEachArgument && forEachBody) {
441+
let toArrayMatch = forEachBody.match(toArrayPattern);
442+
if (toArrayMatch && toArrayMatch.length > 0) {
443+
//let toArray = []
444+
let toArrayBodyField = toArrayMatch[1].split(",");
445+
logger.debug({ toArrayMatch, toArrayBodyField })
446+
finalArray[index] = "input['" + forEachArgument + "'].map(o=> [o['" + toArrayBodyField[0] + "']"
447+
//if(toArrayBodyField.length>2)
448+
if (toArrayBodyField.length > 2)
449+
for (let index = 1; index < toArrayBodyField.length - 1; index++)
450+
finalArray[index] += ", o['" + toArrayBodyField[index] + "']]"
451+
else
452+
finalArray[index] += ", o['" + toArrayBodyField[1] + "']"
453+
finalArray[index] += "])"
454+
}
455+
}
456+
else finalArray[index] = "";
457+
}
458+
else {
499459

500460
var splittedDot = value.match(dotPattern);
501461
if (splittedDot) {
@@ -511,6 +471,7 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
511471
});
512472

513473
// print Array String as output
474+
logger.debug({ finalArray })
514475
resultString = '[';
515476
finalArray.forEach(function (value, index) {
516477
if (value.startsWith("input")) {
@@ -519,11 +480,16 @@ const handleSourceFieldsToDestArray = (sourceFieldArray) => {
519480
} else { //static
520481
resultString += '"' + value + '",';
521482
}
483+
//if (foreachIndex.includes(index)) resultString = resultString.substring(1, resultString.length - 1)
522484
});
523485

524486
resultString = resultString.slice(0, resultString.length - 1) + ']';
525-
return resultString;
526-
} else return '[]';
487+
logger.debug({ resultString })
488+
if (foreachFound)
489+
return resultString.substring(1, resultString.length - 1);
490+
return resultString//.substring(1, resultString.length - 1);
491+
}
492+
else return '[]';
527493
};
528494

529495
/* Returns array notation from dotten notation (without input)

0 commit comments

Comments
 (0)