The data from which the categories are extracted + * if not predefined in the list of categorical features. + * + * @param partialKey: same record structure as the key (see below). + *
Mapping between feature names and categories. + * Some names are mapped to empty categories such that + * their categories could be extracted from dataForUndefinedCategories. + * Names which are mapped to non-empty categories will be assigned the same categories. + * + * @return key: DATASET(KeyLayout) + *
The full mapping between categorical feature names and their categories. + * Its record structure has the following format: + *
+ *
+ * KeyLayout := RECORD + * SET OF STRING+ */ + EXPORT GetKey(dataForUndefinedCategories, partialKey) := FUNCTIONMACRO + IMPORT ML_Core; + + Utl := ML_Core.Preprocessing.Utils; + + KeyLayout := RECORDOF(partialKey); + #EXPORTXML(KeyMetaInfo, partialKey) + dta := #TEXT(dataForUndefinedCategories); + + KeyLayout completeKey(KeyLayout L) := TRANSFORM + #FOR(KeyMetaInfo) + #FOR(field) + #EXPAND('SELF.' + %'@label'% + ' := IF(EXISTS(L.' + %'@label'% + '), ' + + 'L.' + %'@label'% + ',' + + 'Utl.GetCategories(' + dta + ',' + %'@label'% + '))'); + + #END + #END + END; + + Result := PROJECT(partialKey, completeKey(LEFT)); + RETURN Result; + ENDMACRO; + +/** + * Builds a lookup table that maps each category of a feature to a unique number. + * Each category is assigned its index in the category set. + * + * @param key: DATASET(KeyLayout). + *; + * SET OF STRING ; + * ... + * SET OF STRING ; + * END; + *
Mapping between feature names and categories. + * + * @return categoriesMapping: DATASET(MappingLayout). + *
A table with each feature name mapped to its categories and each category + * mapped to its value. + * + *
+ * //record mapping a category to its value. + * Category := RECORD + * STRING categoryName; + * INTEGER value; + * END; + * + * //record mapping feature names to their categories. + * MappingLayout := RECORD + * STRING featureName; + * DATASET(Category) categories; + * END; + *+ */ + EXPORT GetMapping(key) := FUNCTIONMACRO + IMPORT ML_Core; + + RETURN ML_Core.LabelEncoder.MapCategoriesToValues(key); + ENDMACRO; + + /** + * Replaces each categorical value in the data with its index in the key. + * Every unknown category (not in the key) is replaced by -1. + * + * @param dataToEncode: any dataset. + *
The data to encode. + * + * @param key: DATASET(KeyLayout). + *
Mapping between feature names and their categories. + * + * @return encodedData: same record structure as dataToEncode + * with the datatype of all categorical features changed to INTEGER. + *
Data with categorical values replaced by numbers. + */ + EXPORT Encode(dataToEncode, key) := FUNCTIONMACRO + IMPORT ML_Core; + + utils := ML_Core.Preprocessing.Utils; + + //build mapping between categories and values + #UNIQUENAME(mapping) + %mapping% := Utils.LabelEncoder.MapCategoriesToValues(key); + + //build final record structure + featureNameSET := Utils.GetFeatureNames(key); + + #EXPORTXML(dataMetaInfo, RECORDOF(dataToEncode)) + EncodedDataLayout := RECORD + #FOR(dataMetaInfo) + #FOR(field) + #IF(%'@label'% IN featureNameSET) + #EXPAND('INTEGER ' + %'@label'%); + #ELSE + #EXPAND(%'@type'% + ' ' + %'@label'%); + #END + #END + #END + END; + + //replace categories by corresponding value + #EXPORTXML(keyMetaInfo, RECORDOF(key)) + #UNIQUENAME(categories) + #UNIQUENAME(category) + EncodedDataLayout replace (RECORDOF(dataToEncode) L):= TRANSFORM + #FOR(keyMetaInfo) + #FOR(field) + #SET(categories, %'mapping'% + '(featureName = \'' + %'@label'% + '\')[1].categories') + #SET(category, %'categories'% + '(categoryName = (STRING)L.' + %'@label'% + ')') + SELF.%@label% := IF(EXISTS(%category%), %category%[1].value, -1); + #END + #END + SELF := L; + END; + + result := PROJECT(dataToEncode, replace(LEFT)); + RETURN result; + ENDMACRO; + + /** + * Converts back the categorical values into their original labels. + * Every -1 is replaced by an empty string. + * + * @param dataToDecode: any dataset. + *
The data to decode. + * + * @param key: DATASET(KeyLayout). + *
Mapping between feature names and their categories. + * + * @return decodedData: same record structure as dataToDecode + * with the datatype of all categorical features changed to STRING. + *
Data with categorical values replaced by their original labels. + */ + EXPORT Decode(dataToDecode, encoderKey) := FUNCTIONMACRO + IMPORT ML_Core; + + utils := ML_Core.Preprocessing.Utils; + + //build mapping between categories and values + #UNIQUENAME(mapping) + %mapping% := Utils.LabelEncoder.MapCategoriesToValues(key); + + //build final record structure + featureNameSET := Utils.GetFeatureNames(key); + + #EXPORTXML(dataMetaInfo, RECORDOF(dataToDecode)) + DecodedDataLayout := RECORD + #FOR(dataMetaInfo) + #FOR(field) + #IF(%'@label'% IN featureNameSET) + #EXPAND('STRING ' + %'@label'%); + #ELSE + #EXPAND(%'@type'% + ' ' + %'@label'%); + #END + #END + #END + END; + + //replace values by original labels + #EXPORTXML(keyMetaInfo, RECORDOF(key)) + #UNIQUENAME(categories) + #UNIQUENAME(category) + DecodedDataLayout replace (RECORDOF(dataToDecode) L):= TRANSFORM + #FOR(keyMetaInfo) + #FOR(field) + #SET(categories, %'mapping'% + '(featureName = \'' + %'@label'% + '\')[1].categories') + #SET(category, %'categories'% + '(value = L.' + %'@label'% + ')') + SELF.%@label% := %category%[1].categoryName; + #END + #END + SELF := L; + END; + + result := PROJECT(dataToDecode, replace(LEFT)); + RETURN result; + ENDMACRO; +END; diff --git a/Preprocessing/MinMaxScaler.ecl b/Preprocessing/MinMaxScaler.ecl new file mode 100644 index 0000000..bd21888 --- /dev/null +++ b/Preprocessing/MinMaxScaler.ecl @@ -0,0 +1,123 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved. +############################################################################## */ + +IMPORT $.^ as ML_Core; + +Types := ML_Core.Preprocessing.Types; +KeyLayout := Types.MinMaxScaler.KeyLayout; +FeatureMinMax := Types.MinMaxScaler.FeatureMinMax; +NumericField := ML_Core.types.NumericField; +t_FieldReal := ML_Core.types.t_FieldReal; + +/** + * shift the values in a range [min, max]. + * + * @param baseData: DATASET(NumericField), Default = DATASET([], NumericField). + *
The data from which the minimums and maximums are determined. + * + * @param low: t_FieldReal, Default = 0.0 + *
The minimum value of the normalized data. + * + * @param high: t_FieldReal, Default = 1.0 + *
The maximum value of the normalized data. + * + * @param key: DATASET(KeyLayout), default = DATASET([], KeyRec). + *
The key to be reused for scaling/unscaling. + */ +EXPORT MinMaxScaler (DATASET(NumericField) baseData = DATASET([], NumericField), + t_FieldReal lowBound = 0.0, t_FieldReal highBound = 1.0, + DATASET(KeyLayout) key = DATASET([], KeyLayout)) := MODULE + + /** + * Get mins and maxs for each feature in baseData. + * + * @return minAndMaxByFeature: DATASET(KeyLayout). + */ + SHARED ComputeKey() := FUNCTION + //compute the mins and max for each feature + FeatureMinMax GetMinAndMax(Types.numberLayout L) := TRANSFORM + SELF.featureId := L.number; + values := SET(baseData(number = L.number), value); + SELF.minValue := MIN(values); + SELF.maxValue := MAX(values); + END; + + featureIds := DATASET(SET(baseData(id = 1), number), Types.numberLayout); + minsAndMaxs := PROJECT(featureIds, GetMinAndMax(LEFT)); + + //add lowBound and highBound to key + Result := DATASET([{lowBound, highBound, minsAndMaxs}], KeyLayout); + boundariesErrorMsg := 'lowBound must be strictly smaller than high bound'; + RETURN IF(lowBound < highBound, Result, ERROR(KeyLayout, 2, boundariesErrorMsg)); + END; + + //the key used by encode and decode functions + SHARED errorMsg := 'MinMaxScaler: must pass either baseData or key!'; + SHARED innerKey := IF(EXISTS(key), + key, + IF(EXISTS(baseData), + ComputeKey(), + ERROR(KeyLayout, 1, errorMsg))); + + + /** + * Computes the key or reuses it if already given. + * + * @return the key: DATASET(KeyLayout). + */ + EXPORT GetKey() := FUNCTION + RETURN innerKey; + END; + + + /** + * scale the data using the following formula: + * x' = min + ([(x - x_min)(max - min)]/(x_max - x_min)) + * + * @param dataToScale: DATASET(NumericField) . + *
The data to scale. + * + * @return the scaled data: DATASET(NumericField) + */ + EXPORT Scale (DATASET(NumericField) dataToScale) := FUNCTION + IMPORT STD; + + low := innerKey[1].lowBound; + high := innerKey[1].highBound; + + NumericField XF(NumericField L) := TRANSFORM + minValue := innerKey.minsMaxs(featureId = L.number)[1].minValue; + maxValue := innerKey.minsMaxs(featureId = L.number)[1].maxValue; + SELF.value := low + (((L.value - minValue) * (high - low))/(maxValue - minValue)); + SELF := L; + END; + + scaledData := PROJECT(dataToScale, XF(LEFT)); + RETURN scaledData; + END; + + /** + * unscale the data using the following formula + * x = x_min + ((x' - min)(x_max - x_min))/(max-min) + * + * @param dataToUnscale: DATASET(NumericField) + *
The data to unscale. + * + * @return the unscaled data: DATASET(NumericField). + */ + EXPORT unscale(DATASET(NumericField) dataToUnscale) := FUNCTION + low := innerKey[1].lowBound; + high := innerKey[1].highBound; + + NumericField XF(NumericField L) := TRANSFORM + minValue := innerKey.minsMaxs(featureId = L.number)[1].minValue; + maxValue := innerKey.minsMaxs(featureId = L.number)[1].maxValue; + SELF.value := minValue + (((L.value - low) * (maxValue - minValue))/(high - low)); + SELF := L; + END; + + unscaledData := PROJECT(dataToUnscale, XF(LEFT)); + RETURN unscaledData; + END; +END; \ No newline at end of file diff --git a/Preprocessing/Normaliz.ecl b/Preprocessing/Normaliz.ecl new file mode 100644 index 0000000..8beb8ea --- /dev/null +++ b/Preprocessing/Normaliz.ecl @@ -0,0 +1,148 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved. +############################################################################## */ + +IMPORT $.^ as ML_Core; + +PTypes := ML_Core.Preprocessing.Types; +normsLayout := ML_Core.Preprocessing.Types.Normaliz.NormsLayout; +NumericField := ML_Core.Types.NumericField; +t_FieldReal := ML_Core.Types.t_FieldReal; + +/** + * Compute L1 norm. + *
Given a set of values, the L1 norm is the sum of absolute values. + * + * @param values: SET OF t_FieldReal. + *
The set of values from which the l1 norm will be determined. + * + * @return l1Norm: t_FieldReal. + */ +GetL1Norm(SET OF t_FieldReal values) := FUNCTION + LoopLayout := RECORD + UNSIGNED cnt; + t_FieldReal sumOfAbs; + END; + + initialResult := DATASET([{2, ABS(values[1])}], LoopLayout); + loopResult := LOOP(initialResult, + COUNT(values) - 1, + PROJECT(ROWS(LEFT), + TRANSFORM(LoopLayout, + SELF.sumOfAbs := LEFT.sumOfAbs + ABS(values[LEFT.cnt]); + SELF.cnt := LEFT.cnt + 1 ))); + + l1Norm := loopResult[1].sumOfAbs; + RETURN l1Norm; +END; + +/** + * Compute L2 norm. + *
Given a set of values, the L2 norm is the square root of the sum of squares. + * + * @param values: SET OF t_FieldReal. + *
The set of values from which the l2 norm will be determined. + * + * @return l2Norm: t_FieldReal. + */ +GetL2Norm(SET OF t_FieldReal values) := FUNCTION + LoopLayout := RECORD + UNSIGNED cnt; + t_FieldReal sumOfSquares; + END; + + initialResult := DATASET([{2, POWER(values[1], 2)}], LoopLayout); + loopResult := LOOP(initialResult, + COUNT(values) - 1, + PROJECT(ROWS(LEFT), + TRANSFORM(LoopLayout, + SELF.sumOfSquares := LEFT.sumOfSquares + POWER(values[LEFT.cnt], 2); + SELF.cnt := LEFT.cnt + 1))); + + l2Norm := SQRT(loopResult[1].sumOfSquares); + RETURN l2Norm; +END; + +/** + * Compute L-Infinty norm. + *
Given a set of values the l-infinty norm is the value with highest absolute value. + * + * @param values: SET OF t_FieldReal. + *
the set of values from which the l-infinity norm will be determined. + * + * @return lInfintyNorm: t_FieldRead. + */ +GetLInfinityNorm(SET OF t_FieldReal values) := FUNCTION + LoopLayout := RECORD + UNSIGNED cnt; + t_FieldReal maxMagnitude; + END; + + //set initial maxMagnitude to absolute value of first value. + initialResult := DATASET([{2, ABS(values[1])}], LoopLayout); + + //looping through the remaining values to find the maximum magnitude. + loopResult := LOOP(initialResult, + COUNT(values) - 1, + PROJECT(ROWS(LEFT), + TRANSFORM(LoopLayout, + SELF.maxMagnitude := IF(LEFT.maxMagnitude < ABS(values[LEFT.cnt]), + ABS(values[LEFT.cnt]), + LEFT.maxMagnitude); + SELF.cnt := LEFT.cnt + 1))); + + lInfintyNorm := loopResult[1].maxMagnitude; + RETURN lInfintyNorm; +END; + +/** + * computes the norm for each row in the data. + * + * @param dataToNormalize: DATASET(Types.NumericField). + *
The data to normalize. + * + * @param norm: STRING3. + *
The norm based on which the data will be normalized. + * + * @return norms: DATASET(normsLayout). + */ +ComputeNorms(DATASET(NumericField) dataToNormalize, STRING3 norm) := FUNCTION + normsLayout ComputeNorm (PTypes.idLayout L) := TRANSFORM + SELF.id := L.id; + values := SET(dataToNormalize(id = L.id), value); + invalidNormError := ERROR('Invalid norm! Norm must be \'l1\', \'l2\' or \'inf\''); + SELF.value := IF(norm = 'l1', GetL1Norm(values), + IF(norm = 'l2', GetL2Norm(values), + IF(norm = 'inf', GetLInfinityNorm(values), invalidNormError))); + END; + + ids := DATASET(SET(dataToNormalize, id), PTypes.idLayout); + uniqueIDs := DEDUP(ids); + norms := PROJECT(uniqueIDs, ComputeNorm(LEFT)); + RETURN norms; +END; + +/** + * Allows to normalize data based on L1 norm, L2 norm or L-Infinty norm. + * + * @param dataToNormalize: DATASET(Types.NumericField) + *
The data to normalize. + * + * @param norm: STRING3, Default = 'l2'. + *
The norm based on which the data will be normalized. + *
valid values: 'l1', 'l2', 'inf'. + * + * @return the normalizedData: DATASET(NumericField). + */ +EXPORT Normaliz (DATASET(NumericField) dataToNormalize, STRING3 norm = 'l2') := FUNCTION + norms_ := ComputeNorms(dataToNormalize, norm); //compute the norms + + //apply norm on each value + NumericField normalizeRow (NumericField L) := TRANSFORM + SELF.value := L.value/norms_(id = L.id)[1].value; + SELF := L; + END; + + normalizedData := PROJECT(dataToNormalize, normalizeRow(LEFT)); + RETURN normalizedData; +END; diff --git a/Preprocessing/OneHotEncoder.ecl b/Preprocessing/OneHotEncoder.ecl new file mode 100644 index 0000000..656141d --- /dev/null +++ b/Preprocessing/OneHotEncoder.ecl @@ -0,0 +1,263 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved. +############################################################################## */ + +/** + * For categorical features within a dataset, + * this module creates new binary and numeric features. The new binary features only takes a value of 0 or 1 indicating the true value of the category. + * + * @param baseData: DATASET(NumericField), default = DATASET([], NumericField). + *
Data from which categories are extracted for encoding/decoding. + * + * @param featureIds: SET OF UNSIGNED, default = []. + *
The set of indexes (numbers) of categorical features. + * + * @return key: DATASET(KeyLayout), default = DATASET([], KeyLayout) + *
A dataset storing, for each feature in baseData, its index (number), + * its start index in the encoded data, and its category set. + */ + +IMPORT $.^ as ML_Core; + +NumericField := ML_Core.Types.NumericField; +KeyLayout := $.Types.OneHotEncoder.KeyLayout; + + +EXPORT OneHotEncoder(DATASET(NumericField) baseData = DATASET([], NumericField), + SET OF UNSIGNED featureIds = [], + DATASET(KeyLayout) key = DATASET([], KeyLayout)) := MODULE + + SHARED numberLayout := $.Types.numberLayout; + SHARED valueLayout := $.Types.valueLayout; + SHARED t_fieldNumber := ML_Core.Types.t_FieldNumber; + + /** + * Validates input. + *
Input is valid if either baseData and featureIds are provided or key is provided. + * + * @return True when input is valid, False otherwise. + */ + EXPORT isValidInput() := FUNCTION + Result := IF(EXISTS(key) OR (EXISTS(baseData) AND EXISTS(featureIds)), True, False); + RETURN Result; + END; + + /** + * Extracts from base data, the number and categories of each feature in featureIds. + * + * @return partialKey: DATASET(KeyLayout). + *
A dataset storing, for each feature in baseData, its index (number) and its category + * set. + */ + EXPORT GetNumberAndCategories() := FUNCTION + IMPORT STD; + + featureIdDS := DATASET(featureIds, numberLayout); + + KeyLayout extract(numberLayout L) := TRANSFORM + SELF.number := L.number; + + valueSET := SET(baseData(number = L.number), value); + valueDS := SORT(DATASET(valueSET, valueLayout), value); + isCategorical := EXISTS(featureIdDS(number = L.number)); + SELF.categories := IF(isCategorical, DEDUP(valueDS), DATASET([], valueLayout)); + SELF.startNumWhenEncoded := 0; + END; + + featureNumbers := DATASET(SET(baseData(id = 1), number), numberLayout); + Result := PROJECT(featureNumbers, extract(LEFT)); + RETURN Result; + END; + + /** + * Computes the key using basedData and featureIds. + * + * @return keyFromData: DATASET(KeyLayout) + *
A dataset storing, for each feature in baseData, its index (number), + * its start index in the encoded data, and its category set. + */ + EXPORT GetKeyFromData() := FUNCTION + //iterate over each consecutive pair of features to compute the start number in encoded data + KeyLayout computeStartNum (KeyLayout L, KeyLayout R) := TRANSFORM + //if there are categories, then the gap is the number of categories. Otherwise, it is 1. + gap := IF(EXISTS(L.categories), COUNT(L.categories), 1); + //when L is empty (initially), we set startNumWhenEncoded to 1. + SELF.startNumWhenEncoded := IF(L.startNumWhenEncoded = 0, 1, L.startNumWhenEncoded + gap); + SELF := R; + END; + + //we first get the number and categories and then iterate through that result to get the start + //numbers in encoded data + partialKey := GetNumberAndCategories(); + Result := ITERATE(partialKey, computeStartNum(LEFT, RIGHT)); + RETURN Result; + END; + + //the key used by encode and decode functions + errorMsg := 'OneHotEncoder: must pass either key or baseData and featureIds!'; + SHARED innerKey := IF(isValidInput(), + IF(EXISTS(key), key, GetKeyFromData()), + ERROR(KeyLayout, 1, errorMsg)); + + //the FeatureIds used by encode and decode functions + SHARED innerFeatureIds := IF (EXISTS(key), SET(key(EXISTS(categories)), number), featureIds); + + /** + * Extracts, from base data, the index, start number in encoded data and categories of each + * feature in baseData or returns the key if it was provided. If a feature is not included in + * featureIds, its categories will be empty. + * + * @return key: DATASET(KeyLayout) + *
A dataset storing, for each feature in baseData, its index (number), + * its start index in the encoded data, and its category set. + */ + EXPORT GetKey() := FUNCTION + RETURN innerKey; + END; + + /** + * oneHotEncodes a single row (value of a feature). + * + * @param row_: NumericField. + *
The row to oneHotEncode. + * + * @return the row's oneHotCode: DATASET(NumericField). + *
The oneHotCode of the row. + */ + SHARED EncodeRow(NumericField row_) := FUNCTION + startNumber := innerKey(number = row_.number)[1].startNumWhenEncoded; + + //assign 1 if row value matches the category, 0 otherwise. + NumericField XF(valueLayout L, t_FieldNumber cnt) := TRANSFORM + SELF.wi := row_.wi; + SELF.id := row_.id; + SELF.number := startNumber + cnt - 1; + SELF.value := IF(L.value = row_.value, 1, 0); + END; + + //feature's categories for the row + categories := innerKey(number = row_.number)[1].categories; + Result := PROJECT(categories, XF(LEFT, COUNTER)); + RETURN Result; + END; + + //record for storing a oneHotCode. + SHARED OneHotCode := RECORD + DATASET(NumericField) value; + END; + + /** + * OneHotEncodes a given dataset. + * + * @param dataToEncode: DATASET(NumericField). + *
The data to encode. + * + * @return encodedData: DATASET(NumericField). + *
The oneHotEncoded data. + */ + EXPORT Encode (DATASET(NumericField) dataToEncode) := FUNCTION + //convert each row in its oneHotCode + OneHotCode convert (NumericField L) := TRANSFORM + numIfNonCategorical := innerKey(number = L.number)[1].startNumWhenEncoded; + SELF.value := IF(L.number IN innerFeatureIds, + EncodeRow(L), + DATASET([{L.wi, L.id, numIfNonCategorical, L.value}], NumericField)); + END; + + oneHotCodes := PROJECT(dataToEncode, convert(LEFT)); + + //Get all oneHotCodes as a single dataset of NumericField + NumericField XF(NumericField R) := TRANSFORM + SELF := R; + END; + + Result := NORMALIZE(oneHotCodes, LEFT.value, XF(RIGHT)); + RETURN Result; + END; + + /** + * Build a table mapping all field ids (number field) in onehotencoded data + * to the field id to which they correspond to in the original data. + * + * @param numbersInEncodedData: + *
The unique numbers (number field) in the encoded data. + * + * @return mapping: DATASET(numberMapping). + *
The mapping between the numbers. + */ + EXPORT GetNumberMapping(DATASET(numberLayout) numbersInEncodedData) := FUNCTION + numberMapping := $.Types.OneHotEncoder.numberMapping; + + startNums := SET(innerKey, startNumWhenEncoded); + + //setup result template using innerkey, setting all unknowns to 0 + numberMapping setup (numberLayout L) := TRANSFORM + SELF.numberWhenEncoded := L.number; + SELF.numberWhenDecoded := IF(L.number IN startNums, + innerKey(startNumWhenEncoded = L.number)[1].number, + 0); + END; + + resultTemplate := PROJECT(numbersInEncodedData, setup(LEFT)); + + //replace all zeros by the numberWhenDecoded of previous + numberMapping replaceZeros (numberMapping L, numberMapping R) := TRANSFORM + SELF.numberWhenDecoded := IF(L.numberWhenEncoded <> 0 AND R.numberWhenDecoded = 0, + L.numberWhenDecoded, + R.numberWhenDecoded); + SELF := R; + END; + + Result := ITERATE(resultTemplate, replaceZeros(LEFT, RIGHT)); + RETURN Result; + END; + + /** + * Decodes a oneHotEncoded data. + * + * @param dataToDecode: DATASET(NumericField). + *
The data to decode. + * + * @return decodedData: DATASET(NumericField). + *
The decoded data. + */ + EXPORT Decode (DATASET(NumericField) dataToDecode) := FUNCTION + //map numbers in encoded data to numbers in decoded data + numbersInData := SET(dataToDecode(id = 1), number); + numberMapping := GetNumberMapping(DATASET(numbersInData, numberLayout)); + + //append to each row in dataToDecode its number when decoded + ExtendedNumericField := RECORD + NumericField nf; + t_fieldNumber numberWhenDecoded; + END; + + ExtendedNumericField extend(NumericField L) := TRANSFORM + SELF.numberWhenDecoded := numberMapping(numberWhenEncoded = L.number)[1].numberWhenDecoded; + SELF.nf := L; + END; + + extendedData := PROJECT(dataToDecode, extend(LEFT)); + + //rollup grouping by wi, id and numberWhenDecoded + NumericField convert (ExtendedNumericField top, DATASET(ExtendedNumericField) grp) := TRANSFORM + SELF.wi := top.nf.wi; + SELF.id := top.nf.id; + SELF.number := top.numberWhenDecoded; + + //idx in categories = number where value is 1 - number of first + 1 + idx := grp(nf.value = 1)[1].nf.number - top.nf.number + 1; + currentKey := innerKey(startNumWhenEncoded = top.nf.number)[1]; + valueIfcategorical := currentKey.categories[idx].value; + isCategorical := currentKey.number IN innerFeatureIds; + isUnknown := COUNT(grp(nf.value <> 0)) = 0; + SELF.value := IF(isCategorical, + IF(isUnknown, -1, valueIfcategorical), + top.nf.value); + END; + + groupedData := GROUP(extendedData, nf.wi, nf.id, numberWhenDecoded); + Result := ROLLUP(groupedData, GROUP, convert(LEFT, ROWS(LEFT))); + RETURN Result; + END; +END; diff --git a/Preprocessing/README.md b/Preprocessing/README.md new file mode 100644 index 0000000..93a7909 --- /dev/null +++ b/Preprocessing/README.md @@ -0,0 +1,10 @@ +# HPCCSystemsMLPreprocessor + +Machine Learning Preprocessing Bundle for HPCC Systems +- LabelEncoder +- OneHotEncoder +- MinMaxScaler +- StandardScaler +- Normaliz +- Split +- StratifiedSplit diff --git a/Preprocessing/Split.ecl b/Preprocessing/Split.ecl new file mode 100644 index 0000000..2fcd6a8 --- /dev/null +++ b/Preprocessing/Split.ecl @@ -0,0 +1,67 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +IMPORT $.^ as ML_Core; + +Preprocessing := ML_Core.Preprocessing; +Types := Preprocessing.Types; +NumericField := ML_Core.Types.NumericField; + +/** + * Split data into training and test data. + * + * @param dataToSplit: DATASET(Types.NumericField). + *
The data to split. + * + * @param trainSize: REAL4, DEFAULT = 0.0. + *
The training size. If 0.0, it will be set as count(dataToSplit) - testSize. + * + * @param testSize: REAL4, DEFAULT = 0.0. + *
The test size. If 0.0, it will be set as count(dataToSplit) - trainSize. + * + * @param shuffle: Boolean, DEFAULT = false. + *
if true, the data is shuffled before splitting. + * + * @return training and test data + */ +EXPORT Split(DATASET(NumericField) dataToSplit, + REAL4 trainSize = 0.0, + REAL4 testSize = 0.0, + BOOLEAN shuffle = FALSE) := FUNCTION + + ids := DEDUP(DATASET(SET(dataToSplit, id), Types.idLayout)); + + //Get the count of training and test sets + dataCount := COUNT(ids); + trainCount := ROUND(IF(trainSize = 0.0, dataCount * (1-testSize), dataCount * trainSize)); + testCount := ROUND(IF(testSize = 0.0, dataCount * (1-trainSize), dataCount * testSize)); + + //get ids of training and test sets + traininingIds := SET(ids, id)[1..trainCount]; + testIds := SET(ids, id)[(trainCount + 1) .. (trainCount + testCount)]; + + //extract training and test data + ds := IF(shuffle = TRUE, Preprocessing.Utils.Shuffle(dataToSplit), dataToSplit); + trainData := ds(id IN traininingIds); + testData := ds(id IN testIds); + + //fix ids ordering so it starts from 1 + finalTrainData := Preprocessing.Utils.ResetID(trainData); + finalTestData := Preprocessing.Utils.ResetID(testData); + + ResultLayout := RECORD + DATASET(NumericField) trainData; + DATASET(NumericField) testData; + END; + + splitResult := DATASET([{finalTrainData, finalTestData}], ResultLayout); + + //validate input before splitting + validationMsg := $.Utils.ValidateSplitInput(dataToSplit, trainSize, testSize); + Result := IF(validationMsg = 'valid', + splitResult, + ERROR(ResultLayout, validationMsg)); + + RETURN Result[1]; +END; diff --git a/Preprocessing/StandardScaler.ecl b/Preprocessing/StandardScaler.ecl new file mode 100644 index 0000000..0ce954d --- /dev/null +++ b/Preprocessing/StandardScaler.ecl @@ -0,0 +1,102 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved. +############################################################################## */ + +IMPORT $.^ as ML_Core; + +NumericField := ML_Core.Types.NumericField; +KeyLayout := ML_Core.Preprocessing.Types.StandardScaler.KeyLayout; + +/** + * Allow to scale the values such that they have a zero mean and unit variance. + * + * @param baseData: DATASET(NumericField), default = DATASET([], Types.NumericField) + *
The data from which the means and standard deviations are determined for each feature. + * + * @param key: DATASET(KeyLayout), default = DATASET([], KeyRec) + *
The key to be reused for scaling/unscaling. + */ +EXPORT StandardScaler(DATASET(NumericField) baseData = DATASET([], NumericField), + DATASET(KeyLayout) key = DATASET([], KeyLayout)) := MODULE + + SHARED Preprocessing := ML_Core.Preprocessing; + SHARED valueLayout := Preprocessing.Types.valueLayout; + SHARED numberLayout := Preprocessing.Types.numberLayout; + + /** + * Compute averages and standard deviation (stdevs) for each feature in baseData. + * + * @return avgandStdevByFeature: DATASET(KeyLayout). + */ + SHARED ComputeKey() := FUNCTION + KeyLayout compute(numberLayout L) := TRANSFORM + SELF.featureId := L.number; + values := SET(baseData(number = L.number), value); + SELF.avg := AVE(values); + SELF.stdev := SQRT(VARIANCE(DATASET(values, valueLayout), value)); + END; + + featureIds := DATASET(SET(baseData(id = 1), number), numberLayout); + Result := PROJECT(featureIds, compute(LEFT)); + RETURN Result; + END; + + //the key used by encode and decode functions + SHARED errorMsg := 'StandardScaler: must pass either baseData or key!'; + SHARED innerKey := IF(EXISTS(key), + key, + IF(EXISTS(baseData), + ComputeKey(), + ERROR(KeyLayout, 1, errorMsg))); + + /** + * Compute the average and standard deviation per feature or reuses the key if provided. + * + * @return key: DATASET(KeyLayout). + */ + EXPORT GetKey() := FUNCTION + RETURN innerKey; + END; + + /** + * scale the data using the following formula + * x' = (x - avg)/stdev + * + * @param dataToScale: DATASET(NumericField). + *
The data to scale + * + * @return the scaled data: DATASET(NumericField) + */ + EXPORT Scale (DATASET(NumericField) dataToScale) := FUNCTION + NumericField XF(NumericField L) := TRANSFORM + avg := innerKey(featureID = L.number)[1].avg; + stdev := innerKey(featureID = L.number)[1].stdev; + SELF.value := (L.value - avg)/stdev; + SELF := L; + END; + + scaledData := PROJECT(dataToScale, XF(LEFT)); + RETURN scaledData; + END; + + /** + * unscale the data using the following formula: + * x = (x' * stdev) + avg + * + * @param dataToUnscale: DATASET(NumericField). + *
The data to unscale. + * + * @return the unscaled data: DATASET(NumericField). + */ + EXPORT unscale(DATASET(NumericField) dataToUnscale) := FUNCTION + NumericField XF(NumericField L) := TRANSFORM + avg := innerKey(featureId = L.number)[1].avg; + stdev := innerKey(featureId = L.number)[1].stdev; + SELF.value := (L.value * stdev) + avg; + SELF := L; + END; + + unscaledData := PROJECT(dataToUnscale, XF(LEFT)); + RETURN unscaledData; + END; +END; \ No newline at end of file diff --git a/Preprocessing/StratifiedSplit.ecl b/Preprocessing/StratifiedSplit.ecl new file mode 100644 index 0000000..f205c06 --- /dev/null +++ b/Preprocessing/StratifiedSplit.ecl @@ -0,0 +1,199 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +IMPORT $.^ as ML_Core; + +Types := ML_Core.Preprocessing.Types; +Utils := ML_Core.Preprocessing.Utils; +NumericField := ML_Core.Types.NumericField; +valueLayout := Types.valueLayout; + +//layout for keeping count of label values in dataset +YCountsLayout := RECORD + REAL value; + UNSIGNED cnt; +END; + +/** + * Determine, from full data, the count of each value of the field whose number id is yId. + * + * @param ds: DATASET(NumericField). + *
The dataset from which to extract the values' count. + * + * @param yId: UNSIGNED. + *
The id of the field. + * + * @return the values' count in full data: DATASET(YCountsLayout). + */ +GetYCountsInDS(DATASET(NumericField) ds, UNSIGNED yId) := FUNCTION + yValueSET := SET(ds(number = yid), value); + yValues := DEDUP(SORT(DATASET(yValueSET, valueLayout), value)); + + YCountsLayout GetYCounts (valueLayout L) := TRANSFORM + SELF.value := L.value; + SELF.cnt := COUNT(ds(value = L.value AND number = yId)); + END; + + Result := PROJECT(yValues, GetYCounts(LEFT)); + RETURN Result; +END; + +/** + * Determine, from train data, the count of each value of the field whose number id is yId. + * + * @param yCountsInDS: DATASET(YCountsLayout). + *
The y Counts from full data. + * + * @param trainSize: REAL4. + *
The training size. + * + * @return the values' count in train data: DATASET(YCountsLayout). + */ +GetYCountsInTrainDS(DATASET(YCountsLayout) yCountsInDS, REAL4 trainSize) := FUNCTION + YCountsLayout GetYCounts(YCountsLayout L) := TRANSFORM + SELF.value := L.value; + SELF.cnt := ROUND(L.cnt * trainSize); + END; + + Result := PROJECT(yCountsInDS, GetYCounts(LEFT)); + RETURN Result; +END; + +/** + * Determine, from test data, the count of each value of the field whose number id is yId. + * + * @param yCountsInDS: DATASET(YCountsLayout). + *
The y Counts from full data. + * + * @param testSize: REAL4. + *
The test size. + * + * @return the values' count in test data: DATASET(YCountsLayout). + */ +GetYCountsInTestDS(DATASET(YCountsLayout) yCountsInDS, REAL4 testSize) := FUNCTION + YCountsLayout GetYCounts(YCountsLayout L) := TRANSFORM + SELF.value := L.value; + SELF.cnt := ROUND(L.cnt * testSize); + END; + + Result := PROJECT(yCountsInDS, GetYCounts(LEFT)); + RETURN Result; +END; + +/** + * Extracts train and test data based on label field (yid) and + * label field values' count in train and test data. + * + * @param ds: DATASET(NumericField). + *
The dataset from which to extract train and test sets. + * + * @param yid: UNSIGNED. + *
the label id. + * + * @param yCountsInTrainDS: DATASET(YCountsLayout). + *
Label field values' count in train data. + * + * @param yCountsInTestDS: DATASET(YCountsLayout). + *
Label field values' count in test data. + * + * @return training and test data. + */ +GetTrainAndTestData(DATASET(NumericField) ds, UNSIGNED yId, + DATASET(YCountsLayout) yCountsInTrainDS, + DATASET(YCountsLayout) yCountsInTestDS) := FUNCTION + + ResultLayout := RECORD + UNSIGNED cnt; + DATASET(NumericField) trainData; + DATASET(NumericField) testData; + END; + + emptyNF := DATASET([], NumericField); + initialLoopResult := DATASET([{1, emptyNF, emptyNF}], ResultLayout); + maxNumber := MAX(SET(ds, number)); + + Result := LOOP(initialLoopResult, + COUNT(yCountsInTrainDS), + PROJECT(ROWS(LEFT), + TRANSFORM(ResultLayout, + trainCount := yCountsInTrainDS[LEFT.cnt].cnt * maxNumber; + testCount := yCountsInTestDS[LEFT.cnt].cnt * maxNumber; + relevantIds := SET(ds(number = yid AND value = yCountsInTrainDS[LEFT.cnt].value), id); + relevantRows := ds(id IN relevantIds); + SELF.trainData := LEFT.trainData + relevantRows[1..trainCount]; + SELF.testData := LEFT.testData + relevantRows[(trainCount + 1)..(trainCount + testCount)]; + SELF.cnt := LEFT.cnt + 1))); + + RETURN Result; +END; + +/** + * validate yId (must be between 1 and max field id). + */ +validateYID(DATASET(NumericField) ds, UNSIGNED yid) := FUNCTION + maxNumber := MAX(SET(ds, number)); + Result := IF(yid >= 1 AND yid <= maxNumber, 'valid', 'yId valid range = [1, ' + maxNumber + ']'); + RETURN Result; +END; + +/** + * validate ds, trainSize, testSize and yId + */ +validateInput(DATASET(NumericField) ds, REAL4 trainSize, REAL4 testSize, UNSIGNED yId) := FUNCTION + splitValidationMsg := $.Utils.validateSplitInput(ds, trainSize, testSize); + Result := IF(splitValidationMsg = 'valid', validateYID(ds, yId), splitValidationMsg); + RETURN Result; +END; + +/** + * Allows to split data while maintaining the proportions of a feature. + * + * @param ds: DATASET(NumericField). + * The data to split. + * + * @param trainSize: REAL4, Default = 0.0 + *
The training size. + * + * @param testSize: REAL4, Default = 0.0 + *
The test size. + * + * @param labelId: UNSIGNED, Default = 0. + *
The number of the field whose proportions has to be maintained. + * + * @return the training data, test data as DATASET(NumericField). + */ +EXPORT StratifiedSplit(DATASET(NumericField) ds, + REAL4 trainSize = 0, REAL4 testSize = 0, UNSIGNED labelId = 0, + BOOLEAN shuffle = FALSE) := FUNCTION + + yId := IF(labelId = 0, MAX(SET(ds, number)), labelId); + finalTrainSize := IF(trainSize = 0, 1 - testSize, trainSize); + finalTestSize := IF(testSize = 0, 1 - trainSize, testSize); + finalDS := IF(shuffle = TRUE, Utils.Shuffle(ds), ds); + + yCountsInDS := GetYCountsInDS(finalDS, yId); + yCountsInTrainDS := GetYCountsInTrainDS(yCountsInDS, finalTrainSize); + yCountsInTestDS := GetYCountsInTestDS(yCountsInDS, finalTestSize); + trainAndTestData := GetTrainAndTestData(finalDS, yId, yCountsInTrainDS, yCountsInTestDS); + + tempTrainData := SORT(trainAndTestData.trainData, id); + tempTestData := SORT(trainAndTestData.testData, id); + + finalTrainData := Utils.ResetID(tempTrainData); + finalTestData := Utils.ResetID(tempTestData); + + Resultlayout := RECORD + DATASET(NumericField) trainData; + DATASET(NumericField) testData; + END; + + SplitResult := DATASET([{finalTrainData, finalTestData}], Resultlayout); + + validationMsg := validateInput(ds, finalTrainSize, finalTestSize, yId); + Result := IF(validationMsg = 'valid', + SplitResult, + ERROR(Resultlayout, validationMsg)); + + RETURN Result[1]; +END; diff --git a/Preprocessing/Test/Functional/RunFunctionalTests.ecl b/Preprocessing/Test/Functional/RunFunctionalTests.ecl new file mode 100644 index 0000000..ecfdc32 --- /dev/null +++ b/Preprocessing/Test/Functional/RunFunctionalTests.ecl @@ -0,0 +1,14 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +/** + * Run alls the function tests + */ +$.TestLabelEncoder.RunLabelEncoderTests; +$.TestOneHotEncoder.RunOneHotEncoderTests; +$.TestStandardScaler.RunStandardScalerTests; +$.TestMinMaxScaler.RunMinMaxScalerTests; +$.TestNormalize.RunNormalizTests; +$.TestSplit.RunSplitTests; +$.TestUtils.RunUtilsTests; \ No newline at end of file diff --git a/Preprocessing/Test/Functional/TestLabelEncoder/RunLabelEncoderTests.ecl b/Preprocessing/Test/Functional/TestLabelEncoder/RunLabelEncoderTests.ecl new file mode 100644 index 0000000..e876abd --- /dev/null +++ b/Preprocessing/Test/Functional/TestLabelEncoder/RunLabelEncoderTests.ecl @@ -0,0 +1,14 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +/** + * Runs all label encoder functional tests + */ +$.TestGetKey.TestValidInput(); + +$.TestEncode.TestValidInput(); +$.TestEncode.TestUnknownCategories(); + +$.TestDecode.TestValidInput(); +$.TestDecode.TestUnknownCategories(); \ No newline at end of file diff --git a/Preprocessing/Test/Functional/TestLabelEncoder/TestDataAndTypes.ecl b/Preprocessing/Test/Functional/TestLabelEncoder/TestDataAndTypes.ecl new file mode 100644 index 0000000..2c93abb --- /dev/null +++ b/Preprocessing/Test/Functional/TestLabelEncoder/TestDataAndTypes.ecl @@ -0,0 +1,72 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +/** + * Data and Record structures used by TestLabelEncoder Modules + */ +EXPORT TestDataAndTypes := MODULE + //record structure for key. + EXPORT KeyLayout := RECORD + SET OF STRING f1; + SET OF STRING f3; + SET OF STRING f4; + END; + + EXPORT key := ROW({['cat1', 'cat2', 'cat3'], + ['1000', '2000', '3000'], + ['low', 'med', 'high']}, KeyLayout); + + //Record structure for sample data + EXPORT sampleDataLayout := RECORD + UNSIGNED id; + STRING f1; //categorical + UNSIGNED f2; //non-categorical + UNSIGNED f3; //categorical + STRING f4; //categorical + END; + + //sample data + EXPORT sampleData := DATASET([{1, 'cat1', 4, 3000, 'high'}, + {2, 'cat3', 5, 2000, 'med'}, + {3, 'cat2', 6, 1000, 'low'}], sampleDataLayout); + + //sample data + EXPORT sampleData2 := DATASET([{1, 'cat1', 4, 3000, 'highy'}, + {2, '', 5, 2000, 'med'}, + {3, 'cat2', 6, 1000, 'low'}], sampleDataLayout); + + //record structure for encoded data + EXPORT EncodedLayout := RECORD + UNSIGNED id; + INTEGER f1; //categorical + UNSIGNED f2; //non-categorical + INTEGER f3; //categorical + INTEGER f4; //categorical + END; + + EXPORT encodedData1 := DATASET([{1, 0, 4, 2, 2}, + {2, 2, 5, 1, 1}, + {3, 1, 6, 0, 0}], EncodedLayout); + + EXPORT encodedData2 := DATASET([{1, 0, 4, 2, -1}, + {2, -1, 5, 1, 1}, + {3, 1, 6, 0, 0}], EncodedLayout); + + //record structure for encoded data + EXPORT DecodedLayout := RECORD + UNSIGNED id; + STRING f1; //categorical + UNSIGNED f2; //non-categorical + STRING f3; //categorical + STRING f4; //categorical + END; + + EXPORT decodedData1 := DATASET([{1, 'cat1', 4, '3000', 'high'}, + {2, 'cat3', 5, '2000', 'med'}, + {3, 'cat2', 6, '1000', 'low'}], DecodedLayout); + + EXPORT decodedData2 := DATASET([{1, 'cat1', 4, 3000, ''}, + {2, '', 5, 2000, 'med'}, + {3, 'cat2', 6, 1000, 'low'}], DecodedLayout); +END; diff --git a/Preprocessing/Test/Functional/TestLabelEncoder/TestDecode.ecl b/Preprocessing/Test/Functional/TestLabelEncoder/TestDecode.ecl new file mode 100644 index 0000000..3a36a32 --- /dev/null +++ b/Preprocessing/Test/Functional/TestLabelEncoder/TestDecode.ecl @@ -0,0 +1,39 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +/** + * Test Decode in LabelEncoder + */ +IMPORT $.^.^.^.^ as ML_Core; + +Encoder := ML_Core.Preprocessing.LabelEncoder; +Comparator := ML_Core.Preprocessing.Utils.DatasetComparator; + +EXPORT TestDecode:= MODULE + SHARED encodedData1 := $.TestDataAndTypes.encodedData1; + SHARED encodedData2 := $.TestDataAndTypes.encodedData2; + SHARED decodedData1 := $.TestDataAndTypes.decodedData1; + SHARED decodedData2 := $.TestDataAndTypes.decodedData2; + SHARED key := $.TestDataAndTypes.key; + + /** + * Test decode with valid input + */ + EXPORT TestValidInput() := FUNCTION + result := encoder.decode(encodedData1, key); + expected := decodedData1; + cmp := Comparator.Compare(result, expected); + RETURN ASSERT(cmp = 0, cmp + ':TestValidInput Failed!'); + END; + + /** + * Test decode with data having unknown categories (encoded as -1) + */ + EXPORT TestUnknownCategories() := FUNCTION + result := encoder.decode(encodedData2, key); + expected := decodedData2; + cmp := Comparator.Compare(result, expected); + RETURN ASSERT(cmp = 0, cmp + ':TestUnknownCategories Failed!'); + END; +END; \ No newline at end of file diff --git a/Preprocessing/Test/Functional/TestLabelEncoder/TestEncode.ecl b/Preprocessing/Test/Functional/TestLabelEncoder/TestEncode.ecl new file mode 100644 index 0000000..a46a5a0 --- /dev/null +++ b/Preprocessing/Test/Functional/TestLabelEncoder/TestEncode.ecl @@ -0,0 +1,35 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +/** + * Test Encode in LabelEncoder + */ +IMPORT $.^.^.^.^ as ML_Core; + +Encoder := ML_Core.Preprocessing.LabelEncoder; +Comparator := ML_Core.Preprocessing.Utils.DatasetComparator; + +EXPORT TestEncode := MODULE + EXPORT sampleData := $.TestDataAndTypes.sampleData; + EXPORT sampleData2 := $.TestDataAndTypes.sampleData2; + EXPORT key := $.TestDataAndTypes.key; + + /** + * Test encode with valid input + */ + EXPORT TestValidInput() := FUNCTION + result := encoder.encode(sampleData, key); + expected := $.TestDataAndTypes.encodedData1; + RETURN ASSERT(Comparator.Compare(result, expected) = 0, 'TestValidInput Failed!'); + END; + + /** + * Test encode with data having unknown categories + */ + EXPORT TestUnknownCategories() := FUNCTION + result := encoder.encode(sampleData2, key); + expected := $.TestDataAndTypes.encodedData2; + RETURN ASSERT(Comparator.Compare(result, expected) = 0, 'TestUnknownCategories Failed!'); + END; +END; \ No newline at end of file diff --git a/Preprocessing/Test/Functional/TestLabelEncoder/TestGetKey.ecl b/Preprocessing/Test/Functional/TestLabelEncoder/TestGetKey.ecl new file mode 100644 index 0000000..85e77a9 --- /dev/null +++ b/Preprocessing/Test/Functional/TestLabelEncoder/TestGetKey.ecl @@ -0,0 +1,44 @@ +/*############################################################################## +## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved. +############################################################################## */ + +IMPORT $.^.^.^.^ as ML_Core; +Encoder := ML_Core.Preprocessing.LabelEncoder; + +/** + * Test GetKey function + */ +EXPORT TestGetKey := MODULE + + EXPORT KeyLayout := $.TestDataAndTypes.KeyLayout; + + /** + * Test whether the keys are the same. + * + * @param k1: DATASET(KeyLayout). + *
the first key. + * + * @param k1: DATASET(KeyLayout). + *
the second key.
+ *
+ * @return True if the keys are equal, false otherwise.
+ */
+ EXPORT areEqualKeys(KeyLayout k1, KeyLayout k2) := FUNCTION
+ Result := IF(k1.f1 = k2.f1 AND k1.f3 = k2.f3 AND k1.f4 = k2.f4, TRUE, FALSE);
+ RETURN Result;
+ END;
+
+ /**
+ * Test GetKey with a valid input
+ */
+ EXPORT testValidInput() := FUNCTION
+ partialKey := ROW({[],[], ['low', 'med', 'high']}, KeyLayout);
+ key := encoder.GetKey($.TestDataAndTypes.sampleData, partialKey);
+ expectedKey := ROW({['cat1', 'cat2', 'cat3'],
+ ['1000', '2000', '3000'],
+ ['low', 'med', 'high']}, KeyLayout);
+
+ result := AreEqualKeys(key, expectedKey);
+ RETURN ASSERT(result = TRUE, 'testValidInput Failed');
+ END;
+END;
diff --git a/Preprocessing/Test/Functional/TestMinMaxScaler/RunMinMaxScalerTests.ecl b/Preprocessing/Test/Functional/TestMinMaxScaler/RunMinMaxScalerTests.ecl
new file mode 100644
index 0000000..000166a
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestMinMaxScaler/RunMinMaxScalerTests.ecl
@@ -0,0 +1,16 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+/**
+ * Run all min max scaler tests
+ */
+$.TestGetKey.TestComputeKey();
+$.TestGetKey.TestKeyReuse();
+$.TestGetKey.TestLowAndHighChange();
+
+$.TestScale.TestValidInput();
+$.TestScale.TestValidInput2();
+
+$.TestUnscale.TestValidInput();
+$.TestUnscale.TestValidInput2();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestMinMaxScaler/TestData.ecl b/Preprocessing/Test/Functional/TestMinMaxScaler/TestData.ecl
new file mode 100644
index 0000000..9da7ba2
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestMinMaxScaler/TestData.ecl
@@ -0,0 +1,60 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Types := ML_Core.Types;
+PTypes := ML_Core.Preprocessing.Types;
+
+/**
+ * Test data for testing standardScaler module
+ */
+EXPORT TestData := MODULE
+ EXPORT sampleData := DATASET([{1, 1, 1, 0},
+ {1, 1, 2, -100.5},
+ {1, 1, 3, -500},
+ {1, 2, 1, 1},
+ {1, 2, 2, -200.5},
+ {1, 2, 3, -250},
+ {1, 3, 1, 2},
+ {1, 3, 2, -300.5},
+ {1, 3, 3, 0},
+ {1, 4, 1, 3},
+ {1, 4, 2, -400.5},
+ {1, 4, 3, 1000}], ML_Core.Types.NumericField);
+
+ EXPORT key1 := DATASET([{0.0, 1.0, [{1, 0, 3},
+ {2, -400.5, -100.5},
+ {3, -500, 1000}]}], PTypes.MinMaxScaler.KeyLayout);
+
+ EXPORT key2 := DATASET([{-100, 100, [{1, 0, 3},
+ {2, -400.5, -100.5},
+ {3, -500, 1000}]}], PTypes.MinMaxScaler.KeyLayout);
+
+ EXPORT scaledData1 := DATASET([{1, 1, 1, 0},
+ {1, 1, 2, 1},
+ {1, 1, 3, 0},
+ {1, 2, 1, 0.33333333},
+ {1, 2, 2, 0.66666667},
+ {1, 2, 3, 0.16666667},
+ {1, 3, 1, 0.66666667},
+ {1, 3, 2, 0.33333333},
+ {1, 3, 3, 0.33333333},
+ {1, 4, 1, 1},
+ {1, 4, 2, 0},
+ {1, 4, 3, 1}], ML_Core.Types.NumericField);
+
+ EXPORT scaledData2 := DATASET([{1, 1, 1, -100},
+ {1, 1, 2, 100},
+ {1, 1, 3, -100},
+ {1, 2, 1, -33.33333333},
+ {1, 2, 2, 33.33333333},
+ {1, 2, 3, -66.66666667},
+ {1, 3, 1, 33.33333333},
+ {1, 3, 2, -33.33333333},
+ {1, 3, 3, -33.33333333},
+ {1, 4, 1, 100},
+ {1, 4, 2, -100},
+ {1, 4, 3, 100}], ML_Core.Types.NumericField);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestMinMaxScaler/TestGetKey.ecl b/Preprocessing/Test/Functional/TestMinMaxScaler/TestGetKey.ecl
new file mode 100644
index 0000000..3f42cc7
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestMinMaxScaler/TestGetKey.ecl
@@ -0,0 +1,48 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+
+/**
+ * Test GetKey
+ */
+EXPORT TestGetKey := MODULE
+ /**
+ * Test compute key
+ */
+ EXPORT TestComputeKey() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData);
+ result := scaler.getKey();
+ expected := $.TestData.key1;
+ both := result & expected;
+ deduped := DEDUP(SORT(both, lowBound, highBound));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestComputeKey failed');
+ END;
+
+ /**
+ * Test reusing key
+ */
+ EXPORT TestKeyReuse() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler(key := $.TestData.key1);
+ result := scaler.getKey();
+ expected := $.TestData.key1;
+ both := result & expected;
+ deduped := DEDUP(SORT(both, lowBound, highBound));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestKeyReuse failed');
+ END;
+
+ /**
+ * Test changing min and max val
+ */
+ EXPORT TestLowAndHighChange() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData, -100, 100);
+ result := scaler.getKey();
+ expected := $.TestData.key2;
+ both := result & expected;
+ deduped := DEDUP(SORT(both, lowBound, highBound));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestLowAndHighChange failed');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestMinMaxScaler/TestScale.ecl b/Preprocessing/Test/Functional/TestMinMaxScaler/TestScale.ecl
new file mode 100644
index 0000000..f19c5af
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestMinMaxScaler/TestScale.ecl
@@ -0,0 +1,35 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test scale
+ */
+EXPORT TestScale := MODULE
+ /**
+ * Test valid input
+ */
+ EXPORT TestValidInput() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData);
+ result := scaler.scale($.TestData.sampleData);
+ expected := $.TestData.scaledData1;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test valid input 2
+ */
+ EXPORT TestValidInput2() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData, -100, 100);
+ result := scaler.scale($.TestData.sampleData);
+ expected := $.TestData.scaledData2;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput2 Failed (' + cmp + ')');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestMinMaxScaler/TestUnscale.ecl b/Preprocessing/Test/Functional/TestMinMaxScaler/TestUnscale.ecl
new file mode 100644
index 0000000..3c6d210
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestMinMaxScaler/TestUnscale.ecl
@@ -0,0 +1,35 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test unscale
+ */
+EXPORT TestUnscale := MODULE
+ /**
+ * Test valid input
+ */
+ EXPORT TestValidInput() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData);
+ result := scaler.unscale($.TestData.scaledData1);
+ expected := $.TestData.sampleData;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test valid input 2
+ */
+ EXPORT TestValidInput2() := FUNCTION
+ scaler := Preprocessing.MinMaxScaler($.TestData.sampleData, -100, 100);
+ result := scaler.unscale($.TestData.scaledData2);
+ expected := $.TestData.sampleData;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput2 Failed (' + cmp + ')');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestNormalize/RunNormalizTests.ecl b/Preprocessing/Test/Functional/TestNormalize/RunNormalizTests.ecl
new file mode 100644
index 0000000..21a65b3
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestNormalize/RunNormalizTests.ecl
@@ -0,0 +1,10 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+/**
+ * Runs all normaliz tests.
+ */
+$.TestNormaliz.TestL1Norm();
+$.TestNormaliz.TestL2Norm();
+$.TestNormaliz.TestLInfNorm();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestNormalize/TestData.ecl b/Preprocessing/Test/Functional/TestNormalize/TestData.ecl
new file mode 100644
index 0000000..9ea09ed
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestNormalize/TestData.ecl
@@ -0,0 +1,62 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+/**
+ * Test data for testing the Normaliz function
+ */
+EXPORT testData := MODULE
+ EXPORT sampleData := DATASET([{1,1,1,4},
+ {1,1,2,1},
+ {1,1,3,2},
+ {1,1,4,2},
+ {1,2,1,1},
+ {1,2,2,3},
+ {1,2,3,9},
+ {1,2,4,3},
+ {1,3,1,5},
+ {1,3,2,7},
+ {1,3,3,5},
+ {1,3,4,1}], ML_Core.Types.NumericField);
+
+ EXPORT l1NormResult := DATASET([{1, 1, 1, 0.44444444},
+ {1, 1, 2, 0.11111111},
+ {1, 1, 3, 0.22222222},
+ {1, 1, 4, 0.22222222},
+ {1, 2, 1, 0.0625},
+ {1, 2, 2, 0.1875},
+ {1, 2, 3, 0.5625},
+ {1, 2, 4, 0.1875},
+ {1, 3, 1, 0.27777778},
+ {1, 3, 2, 0.38888889},
+ {1, 3, 3, 0.27777778},
+ {1, 3, 4, 0.05555556}], ML_Core.Types.NumericField);
+
+ EXPORT l2NormResult := DATASET([{1, 1, 1, 0.8},
+ {1, 1, 2, 0.2},
+ {1, 1, 3, 0.4},
+ {1, 1, 4, 0.4},
+ {1, 2, 1, 0.1},
+ {1, 2, 2, 0.3},
+ {1, 2, 3, 0.9},
+ {1, 2, 4, 0.3},
+ {1, 3, 1, 0.5},
+ {1, 3, 2, 0.7},
+ {1, 3, 3, 0.5},
+ {1, 3, 4, 0.1}], ML_Core.Types.NumericField);
+
+ EXPORT lInfNormResult := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 0.25},
+ {1, 1, 3, 0.5},
+ {1, 1, 4, 0.5},
+ {1, 2, 1, 0.11111111},
+ {1, 2, 2, 0.33333333},
+ {1, 2, 3, 1},
+ {1, 2, 4, 0.33333333},
+ {1, 3, 1, 0.71428571},
+ {1, 3, 2, 1},
+ {1, 3, 3, 0.71428571},
+ {1, 3, 4, 0.14285714}], ML_Core.Types.NumericField);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestNormalize/TestNormaliz.ecl b/Preprocessing/Test/Functional/TestNormalize/TestNormaliz.ecl
new file mode 100644
index 0000000..9401be5
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestNormalize/TestNormaliz.ecl
@@ -0,0 +1,47 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+IMPORT $.TestData;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+IMPORT $.^.^.^.^ as MLC;
+NumericField := MLC.Types.NumericField;
+
+/**
+ * Test the normaliz function
+ */
+EXPORT TestNormaliz := MODULE
+ /**
+ * Test L1 norm
+ */
+ EXPORT TestL1Norm() := FUNCTION
+ result := Preprocessing.Normaliz(testData.sampleData, 'l1');
+ expected := $.testData.l1NormResult;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestL1Norm Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test L2 norm
+ */
+ EXPORT TestL2Norm() := FUNCTION
+ result := Preprocessing.Normaliz(testData.sampleData, 'l2');
+ expected := $.testData.l2NormResult;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestL2Norm Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test L-infinity norm
+ */
+ EXPORT TestLInfNorm() := FUNCTION
+ result := Preprocessing.Normaliz(testData.sampleData, 'inf');
+ expected := $.testData.lInfNormResult;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestLInfNorm Failed (' + cmp + ')');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/RunOneHotEncoderTests.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/RunOneHotEncoderTests.ecl
new file mode 100644
index 0000000..1021e0b
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/RunOneHotEncoderTests.ecl
@@ -0,0 +1,25 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+/**
+ * Runs all OneHotEncoder tests
+ */
+$.TestIsValidInput.TestWhenDataPassed();
+$.TestIsValidInput.TestWhenKeyPassed();
+$.TestIsValidInput.TestWhenAllEmpty();
+$.TestIsValidInput.TestEmptyFeatureIds();
+
+$.TestGetKey.TestValidInput1();
+$.TestGetKey.TestValidInput2();
+$.TestGetKey.TestEmptyInput();
+$.TestGetKey.TestInvalidFeatureID();
+
+$.TestEncode.testKnownCategories();
+$.TestEncode.testEmptyInput();
+$.TestEncode.testUnKnownCategs();
+
+$.TestDecode.TestGetNumberMapping();
+$.TestDecode.TestKnownCategories();
+$.TestDecode.TestUnKnownCategories();
+$.TestDecode.TestEmptyInput();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/TestData.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/TestData.ecl
new file mode 100644
index 0000000..62a5cca
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/TestData.ecl
@@ -0,0 +1,66 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Types := ML_Core.Types;
+PTypes := ML_Core.Preprocessing.Types;
+
+/**
+ * Test Data for Testing OneHotEncoder
+ */
+EXPORT TestData := MODULE
+ SHARED NumericField := Types.NumericField;
+
+ EXPORT validFeatureIds := [1,3];
+ EXPORT invalidFeatureIds := [1,4];
+
+ EXPORT key := DATASET([{1, 1, [{0},{1}]},
+ {2, 3, []},
+ {3, 4, [{1},{3}]}], PTypes.OneHotEncoder.keyLayout);
+
+
+ EXPORT sample1 := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 1000},
+ {1, 1, 3, 3},
+ {1, 2, 1, 0},
+ {1, 2, 2, 2000},
+ {1, 2, 3, 1}], NumericField);
+
+ EXPORT sample2 := DATASET([{1, 1, 1, 2},
+ {1, 1, 2, 1000},
+ {1, 1, 3, 1},
+ {1, 2, 1, 0},
+ {1, 2, 2, 2000},
+ {1, 2, 3, 2}], NumericField);
+
+ EXPORT encodedSample1 := DATASET([{1, 1, 1, 0},
+ {1, 1, 2, 1},
+ {1, 1, 3, 1000},
+ {1, 1, 4, 0},
+ {1, 1, 5, 1},
+ {1, 2, 1, 1},
+ {1, 2, 2, 0},
+ {1, 2, 3, 2000},
+ {1, 2, 4, 1},
+ {1, 2, 5, 0}], NumericField);
+
+ EXPORT encodedSample2 := DATASET([{1, 1, 1, 0},
+ {1, 1, 2, 0},
+ {1, 1, 3, 1000},
+ {1, 1, 4, 1},
+ {1, 1, 5, 0},
+ {1, 2, 1, 1},
+ {1, 2, 2, 0},
+ {1, 2, 3, 2000},
+ {1, 2, 4, 0},
+ {1, 2, 5, 0}], NumericField);
+
+ EXPORT decodedSample2 := DATASET([{1, 1, 1, -1},
+ {1, 1, 2, 1000},
+ {1, 1, 3, 1},
+ {1, 2, 1, 0},
+ {1, 2, 2, 2000},
+ {1, 2, 3, -1}], NumericField);
+END;
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/TestDecode.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/TestDecode.ecl
new file mode 100644
index 0000000..80675f4
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/TestDecode.ecl
@@ -0,0 +1,64 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+NumericField := ML_Core.Types.NumericField;
+
+/**
+ * Test decode.
+ */
+EXPORT TestDecode := MODULE
+ SHARED encoder := Preprocessing.OneHotEncoder(key := $.testData.key);
+
+ /**
+ * Test GetNumberMapping.
+ */
+ EXPORT TestGetNumberMapping() := FUNCTION
+ input := DATASET([{1},{2},{3},{4},{5}], Preprocessing.Types.numberLayout);
+ result := encoder.GetNumberMapping(input);
+ expected := DATASET([{1,1},
+ {2,1},
+ {3,2},
+ {4,3},
+ {5,3}], Preprocessing.Types.OneHotEncoder.NumberMapping);
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestGetNumberMapping Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test decoding data with known categories.
+ */
+ EXPORT TestKnownCategories() := FUNCTION
+ result := encoder.decode($.TestData.encodedSample1);
+ expected := $.TestData.sample1;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestKnownCategories Failed (' + cmp + ')');
+ //RETURN result;
+ END;
+
+ /**
+ * Test decoding data with unknown categories.
+ */
+ EXPORT TestUnKnownCategories() := FUNCTION
+ result := encoder.decode($.TestData.encodedSample2);
+ expected := $.TestData.decodedSample2;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestUnKnownCategories Failed (' + cmp + ')');
+ //RETURN result;
+ END;
+
+ /**
+ * Test decoding empty data.
+ */
+ EXPORT testEmptyInput() := FUNCTION
+ emptyNF := DATASET([], NumericField);
+ result := encoder.decode(emptyNF);
+ expected := emptyNF;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestEmptyData Failed (' + cmp + ')');
+ END;
+END;
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/TestEncode.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/TestEncode.ecl
new file mode 100644
index 0000000..9e9f109
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/TestEncode.ecl
@@ -0,0 +1,49 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+NumericField := ML_Core.Types.NumericField;
+
+/**
+ * Test Encode
+ */
+EXPORT TestEncode := MODULE
+ SHARED sample1 := $.testData.sample1;
+ SHARED sample2 := $.testData.sample2;
+ SHARED key := $.testData.key;
+ SHARED encoder := Preprocessing.OneHotEncoder(key := key);
+ /**
+ * Test with known categories.
+ */
+ EXPORT testKnownCategories() := FUNCTION
+ result := encoder.encode(sample1);
+ expected := $.TestData.encodedSample1;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, cmp + ':testKnownCategories Failed!');
+ END;
+
+ /**
+ * Test with unknown categories.
+ */
+ EXPORT testUnKnownCategs() := FUNCTION
+ result := encoder.encode(sample2);
+ expected := $.TestData.encodedSample2;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, cmp + ':testUnKnownCategs Failed!');
+ END;
+
+ /**
+ * Test encoding empty data
+ */
+ EXPORT testEmptyInput() := FUNCTION
+ emptyNF := DATASET([], NumericField);
+ result := encoder.encode(emptyNF);
+ expected := emptyNF;
+ cmp := Comparator.Compare(result, expected);
+ RETURN ASSERT(cmp = 0, cmp + ':testEmptyInput Failed!');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/TestGetKey.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/TestGetKey.ecl
new file mode 100644
index 0000000..2154aed
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/TestGetKey.ecl
@@ -0,0 +1,84 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^ as Preprocessing;
+
+PTypes := Preprocessing.Types;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test GetKey
+ */
+EXPORT TestGetKey := MODULE
+ SHARED sampleData := $.testData.sample1;
+ SHARED validFeatureIds := $.testData.validFeatureIds;
+ SHARED invalidFeatureIds := $.testData.invalidFeatureIds;
+ SHARED key := $.testData.key;
+ SHARED KeyLayout := PTypes.OneHotEncoder.KeyLayout;
+
+ /**
+ * Test computation with baseData and validFeatureIds.
+ */
+ EXPORT TestValidInput1() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(sampleData, validFeatureIds);
+ result := encoder.getKey();
+ expected := key;
+ both := result & expected;
+ deduped := DEDUP(SORT(both, number));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestValidInput1 Failed!');
+ END;
+
+ /**
+ * Test when key is passed.
+ */
+ EXPORT TestValidInput2() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(key := key);
+ result := encoder.getKey();
+ expected := key;
+ both := result & expected;
+ deduped := DEDUP(SORT(both, number));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestValidInput2 Failed!');
+ END;
+
+ /**
+ * Test when input is empty.
+ */
+ EXPORT TestEmptyInput() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder();
+
+ KeyLayout recoveryXF := TRANSFORM
+ SELF.number := 0;
+ SELF.startNumWhenEncoded := 0;
+ SELF.categories := [];
+ END;
+
+ result := CATCH(encoder.getKey(), ONFAIL(recoveryXF));
+ expected := DATASET([{0,0,[]}], KeyLayout);
+ both := result & expected;
+ deduped := DEDUP(SORT(both, number));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestEmptyInput Failed!');
+ END;
+
+ /**
+ * Test when feature ids are invalid.
+ */
+ EXPORT TestInvalidFeatureID() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(sampleData, invalidFeatureIDs);
+
+ KeyLayout recoveryXF := TRANSFORM
+ SELF.number := 0;
+ SELF.startNumWhenEncoded := 0;
+ SELF.categories := [];
+ END;
+
+ result := CATCH(encoder.getKey(), ONFAIL(recoveryXF));
+ expected := DATASET([{1, 1, [{0},{1}]},
+ {2, 3, []},
+ {3, 4, []}], PTypes.OneHotEncoder.keyLayout);
+
+ both := result & expected;
+ deduped := DEDUP(SORT(both, number));
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestInvalidFeatureID Failed!');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestOneHotEncoder/TestIsValidInput.ecl b/Preprocessing/Test/Functional/TestOneHotEncoder/TestIsValidInput.ecl
new file mode 100644
index 0000000..53ca5f6
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestOneHotEncoder/TestIsValidInput.ecl
@@ -0,0 +1,50 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT Preprocessing;
+
+/**
+ * Test isValidInput
+ */
+EXPORT TestIsValidInput := MODULE
+ SHARED sampleData := $.testData.sample1;
+ SHARED featureIds := $.testData.validFeatureIds;
+ SHARED key := $.testData.key;
+
+ /**
+ * Test when baseData and featureIds are passed
+ */
+ EXPORT TestWhenDataPassed() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(sampleData, featureIds);
+ result := encoder.isValidInput();
+ RETURN ASSERT(result = True, 'TestWhenDataPassed Failed!');
+ END;
+
+ /**
+ * Test when key is passed
+ */
+ EXPORT TestWhenKeyPassed() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(key := key);
+ result := encoder.isValidInput();
+ RETURN ASSERT(result = True, 'TestWhenKeyPassed Failed!');
+ END;
+
+ /**
+ * Test when nothing passed
+ */
+ EXPORT TestWhenAllEmpty() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder();
+ result := encoder.isValidInput();
+ RETURN ASSERT(result = False, 'TestWhenAllEmpty Failed!');
+ END;
+
+ /**
+ * Test when data passed but not featureIds
+ */
+ EXPORT TestEmptyFeatureIds() := FUNCTION
+ encoder := Preprocessing.OneHotEncoder(sampleData);
+ result := encoder.isValidInput();
+ RETURN ASSERT(result = False, 'TestEmptyFeatureIds Failed!');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestSplit/RunSplitTests.ecl b/Preprocessing/Test/Functional/TestSplit/RunSplitTests.ecl
new file mode 100644
index 0000000..593439a
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestSplit/RunSplitTests.ecl
@@ -0,0 +1,10 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+/**
+ * Run split tests.
+ */
+$.TestSplit.TestValidTrainSize();
+$.TestSplit.TestValidTestSize();
+$.TestSplit.TestValidTrainTestSize();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestSplit/TestData.ecl b/Preprocessing/Test/Functional/TestSplit/TestData.ecl
new file mode 100644
index 0000000..679333a
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestSplit/TestData.ecl
@@ -0,0 +1,47 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+NumericField := ML_Core.Types.NumericField;
+
+/**
+ * Test data for testing split function
+ */
+EXPORT testData := MODULE
+ EXPORT sampleData := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 2},
+ {1, 1, 3, 3},
+ {1, 1, 4, 4},
+ {1, 2, 1, 5},
+ {1, 2, 2, 6},
+ {1, 2, 3, 7},
+ {1, 2, 4, 8},
+ {1, 3, 1, 9},
+ {1, 3, 2, 10},
+ {1, 3, 3, 11},
+ {1, 3, 4, 12},
+ {1, 4, 1, 13},
+ {1, 4, 2, 14},
+ {1, 4, 3, 15},
+ {1, 4, 4, 16}], NumericField);
+
+ EXPORT trainData := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 2},
+ {1, 1, 3, 3},
+ {1, 1, 4, 4},
+ {1, 2, 1, 5},
+ {1, 2, 2, 6},
+ {1, 2, 3, 7},
+ {1, 2, 4, 8}], NumericField);
+
+ EXPORT testData := DATASET([{1, 1, 1, 9},
+ {1, 1, 2, 10},
+ {1, 1, 3, 11},
+ {1, 1, 4, 12},
+ {1, 2, 1, 13},
+ {1, 2, 2, 14},
+ {1, 2, 3, 15},
+ {1, 2, 4, 16}], NumericField);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestSplit/TestSplit.ecl b/Preprocessing/Test/Functional/TestSplit/TestSplit.ecl
new file mode 100644
index 0000000..81a96cf
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestSplit/TestSplit.ecl
@@ -0,0 +1,51 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+IMPORT $.TestData;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test Split function.
+ */
+EXPORT TestSplit := MODULE
+ /**
+ * Test with valid train size = 0.5.
+ */
+ EXPORT TestValidTrainSize() := FUNCTION
+ splitResult := Preprocessing.Split(testData.sampleData, 0.5);
+ expectedTrain := splitResult.trainData;
+ expectedTest := splitResult.testData;
+ cmp1 := comparator.compare(expectedTrain, testData.trainData);
+ cmp2 := comparator.compare(expectedTest, testData.testData);
+ RETURN ASSERT(cmp1 = 0 AND cmp2 = 0, 'TestValidTrainSize Failed (' + cmp1 + '|' + cmp2 + ')');
+ END;
+
+ /**
+ * Test with valid test size = 0.5.
+ */
+ EXPORT TestValidTestSize() := FUNCTION
+ splitResult := Preprocessing.Split(testData.sampleData,, 0.5);
+ expectedTrain := splitResult.trainData;
+ expectedTest := splitResult.testData;
+ cmp1 := comparator.compare(expectedTrain, testData.trainData);
+ cmp2 := comparator.compare(expectedTest, testData.testData);
+ RETURN ASSERT(cmp1 = 0 AND cmp2 = 0, 'TestValidTestSize Failed (' + cmp1 + '|' + cmp2 + ')');
+ END;
+
+ /**
+ * Test with valid train and test size.
+ */
+ EXPORT TestValidTrainTestSize() := FUNCTION
+ splitResult := Preprocessing.Split(testData.sampleData,0.5 , 0.5);
+ expectedTrain := splitResult.trainData;
+ expectedTest := splitResult.testData;
+ cmp1 := comparator.compare(expectedTrain, testData.trainData);
+ cmp2 := comparator.compare(expectedTest, testData.testData);
+ errorMsg := 'TestValidTrainTestSize Failed (' + cmp1 + '|' + cmp2 + ')';
+ RETURN ASSERT(cmp1 = 0 AND cmp2 = 0, errorMsg);
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestStandardScaler/RunStandardScalerTests.ecl b/Preprocessing/Test/Functional/TestStandardScaler/RunStandardScalerTests.ecl
new file mode 100644
index 0000000..8bee232
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStandardScaler/RunStandardScalerTests.ecl
@@ -0,0 +1,13 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+/**
+ * Runs all standard scaler tests
+ */
+$.TestGetKey.TestKeyComputation();
+$.TestGetKey.TestKeyReuse();
+
+$.TestScale.testValidInput();
+
+$.TestUnscale.testValidInput();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestStandardScaler/TestData.ecl b/Preprocessing/Test/Functional/TestStandardScaler/TestData.ecl
new file mode 100644
index 0000000..2575116
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStandardScaler/TestData.ecl
@@ -0,0 +1,44 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Types := ML_Core.Types;
+PTypes := ML_Core.Preprocessing.Types;
+
+/**
+ * Test data for testing standardScaler module
+ */
+EXPORT TestData := MODULE
+
+ EXPORT key := DATASET([{1, 1.5, 1.11803399},
+ {2, -250.5, 111.80339887},
+ {3, 62.5, 569.4020987}], PTypes.StandardScaler.keyLayout);
+
+ EXPORT sampleData := DATASET([{1, 1, 1, 0},
+ {1, 1, 2, -100.5},
+ {1, 1, 3, -500},
+ {1, 2, 1, 1},
+ {1, 2, 2, -200.5},
+ {1, 2, 3, -250},
+ {1, 3, 1, 2},
+ {1, 3, 2, -300.5},
+ {1, 3, 3, 0},
+ {1, 4, 1, 3},
+ {1, 4, 2, -400.5},
+ {1, 4, 3, 1000}], Types.NumericField);
+
+ EXPORT scaledData := DATASET([{1, 1, 1, -1.34164079},
+ {1, 1, 2, 1.34164079},
+ {1, 1, 3, -0.98787834},
+ {1, 2, 1, -0.4472136},
+ {1, 2, 2, 0.4472136},
+ {1, 2, 3, -0.5488213},
+ {1, 3, 1, 0.4472136},
+ {1, 3, 2, -0.4472136},
+ {1, 3, 3, -0.10976426},
+ {1, 4, 1, 1.34164079},
+ {1, 4, 2, -1.34164079},
+ {1, 4, 3, 1.6464639}], Types.NumericField);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestStandardScaler/TestGetKey.ecl b/Preprocessing/Test/Functional/TestStandardScaler/TestGetKey.ecl
new file mode 100644
index 0000000..cedc721
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStandardScaler/TestGetKey.ecl
@@ -0,0 +1,57 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test GetKey()
+ */
+EXPORT TestGetKey := MODULE
+ SHARED key := $.TestData.key;
+
+ /**
+ * Test key computation
+ */
+ EXPORT TestKeyComputation() := FUNCTION
+ scaler := Preprocessing.StandardScaler($.testData.sampleData);
+ result := scaler.getKey();
+ expected := key;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestKeyComputation Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test key reuse
+ */
+ EXPORT TestKeyReuse() := FUNCTION
+ scaler := Preprocessing.StandardScaler(key := $.testData.key);
+ result := scaler.getKey();
+ expected := key;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'TestKeyReuse Failed (' + cmp + ')');
+ END;
+
+ /**
+ * Test with empty input
+ */
+ EXPORT TestEmptyInput() := FUNCTION
+ KeyLayout := Preprocessing.Types.StandardScaler.KeyLayout;
+
+ KeyLayout recoveryXF := TRANSFORM
+ SELF.featureId := 0;
+ SELF.avg := 0.0;
+ SELF.stdev := 0.0;
+ END;
+
+ scaler := Preprocessing.StandardScaler();
+ result := CATCH(scaler.getKey(), ONFAIL(recoveryXF));
+ //expected := DATASET([{0,0,0}], KeyLayout);
+ //cmp := Comparator.compare(result, expected);
+ //RETURN ASSERT(cmp = 0, 'TestEmptyInput Failed (' + cmp + ')');
+ RETURN result;
+ END;
+END;
diff --git a/Preprocessing/Test/Functional/TestStandardScaler/TestScale.ecl b/Preprocessing/Test/Functional/TestStandardScaler/TestScale.ecl
new file mode 100644
index 0000000..399becc
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStandardScaler/TestScale.ecl
@@ -0,0 +1,24 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test Scale
+ */
+EXPORT TestScale := MODULE
+ /**
+ * Test valid input
+ */
+ EXPORT testValidInput() := FUNCTION
+ scaler := Preprocessing.StandardScaler($.testData.sampleData);
+ result := scaler.scale($.testData.sampleData);
+ expected := $.testData.scaledData;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput Failed (' + cmp + ')');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestStandardScaler/TestUnscale.ecl b/Preprocessing/Test/Functional/TestStandardScaler/TestUnscale.ecl
new file mode 100644
index 0000000..e006423
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStandardScaler/TestUnscale.ecl
@@ -0,0 +1,24 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+Preprocessing := ML_Core.Preprocessing;
+Comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test UnScale
+ */
+EXPORT TestUnscale := MODULE
+ /**
+ * Test valid input
+ */
+ EXPORT testValidInput() := FUNCTION
+ scaler := Preprocessing.StandardScaler($.testData.sampleData);
+ result := scaler.unscale($.testData.scaledData);
+ expected := $.testData.sampleData;
+ cmp := Comparator.compare(result, expected);
+ RETURN ASSERT(cmp = 0, 'testValidInput Failed (' + cmp + ')');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestStratifiedSplit/Test1.ecl b/Preprocessing/Test/Functional/TestStratifiedSplit/Test1.ecl
new file mode 100644
index 0000000..c5b801d
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStratifiedSplit/Test1.ecl
@@ -0,0 +1,26 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+IMPORT $.TestData;
+
+Preprocessing := ML_Core.Preprocessing;
+comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test case 1
+ * Stratified split provided train size and yId
+ */
+ML_Core.toField(TestData.ds, sampleData);
+
+splitResult := Preprocessing.StratifiedSplit(sampleData, 0.5,,4);
+
+trainData := splitResult.trainData;
+assertMsgTrainDS := 'Train data are different from expected';
+ASSERT(comparator.compare(trainData, testData.expTrainData) = 0, assertMsgTrainDS);
+
+testData_ := splitResult.testData;
+assertMsgTestDS := 'Test data are different from expected';
+ASSERT(comparator.compare(testData_, testData.expTestData) = 0, assertMsgTestDS);
+
diff --git a/Preprocessing/Test/Functional/TestStratifiedSplit/Test2.ecl b/Preprocessing/Test/Functional/TestStratifiedSplit/Test2.ecl
new file mode 100644
index 0000000..2b63cff
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStratifiedSplit/Test2.ecl
@@ -0,0 +1,26 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+IMPORT $.TestData;
+
+Preprocessing := ML_Core.Preprocessing;
+comparator := Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test case 1
+ * Stratified split provided train size, test size
+ */
+ML_Core.toField(TestData.ds, sampleData);
+
+splitResult := Preprocessing.StratifiedSplit(sampleData, 0.5, 0.25);
+
+trainData := splitResult.trainData;
+assertMsgTrainDS := 'Train data are different from expected';
+ASSERT(comparator.compare(trainData, testData.expTrainData) = 0, assertMsgTrainDS);
+
+testData_ := splitResult.testData;
+assertMsgTestDS := 'Test data are different from expected';
+ASSERT(comparator.compare(testData_, testData.expTestData3) = 0, assertMsgTestDS);
+
diff --git a/Preprocessing/Test/Functional/TestStratifiedSplit/TestData.ecl b/Preprocessing/Test/Functional/TestStratifiedSplit/TestData.ecl
new file mode 100644
index 0000000..93869d4
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestStratifiedSplit/TestData.ecl
@@ -0,0 +1,142 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^ as ML_Core;
+
+EXPORT TestData := MODULE
+ EXPORT Layout := RECORD
+ UNSIGNED id;
+ UNSIGNED f1;
+ UNSIGNED f2;
+ UNSIGNED f3;
+ UNSIGNED f4;
+ END;
+
+ EXPORT ds := DATASET([{ 1, 1, 2, 3, 1},
+ { 2, 4, 5, 6, 2},
+ { 3, 7, 8, 9, 3},
+ { 4, 10, 11, 12, 2},
+ { 5, 13, 14, 15, 3},
+ { 6, 16, 17, 18, 2},
+ { 7, 19, 20, 21, 3},
+ { 8, 22, 23, 24, 1},
+ { 9, 25, 26, 27, 2},
+ {10, 28, 29, 30, 1},
+ {11, 31, 32, 33, 1},
+ {12, 34, 35, 36, 3}], Layout);
+
+ EXPORT ds4 := DATASET([{ 1, 1, 2, 1, 3},
+ { 2, 4, 5, 2, 6},
+ { 3, 7, 8, 3, 9},
+ { 4, 10, 11, 2, 12},
+ { 5, 13, 14, 3, 15},
+ { 6, 16, 17, 2, 18},
+ { 7, 19, 20, 3, 21},
+ { 8, 22, 23, 1, 24},
+ { 9, 25, 26, 2, 27},
+ {10, 28, 29, 1, 30},
+ {11, 31, 32, 1, 33},
+ {12, 34, 35, 3, 36}], Layout);
+
+ EXPORT expTrainData := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 2},
+ {1, 1, 3, 3},
+ {1, 1, 4, 1},
+ {1, 2, 1, 4},
+ {1, 2, 2, 5},
+ {1, 2, 3, 6},
+ {1, 2, 4, 2},
+ {1, 3, 1, 7},
+ {1, 3, 2, 8},
+ {1, 3, 3, 9},
+ {1, 3, 4, 3},
+ {1, 4, 1, 10},
+ {1, 4, 2, 11},
+ {1, 4, 3, 12},
+ {1, 4, 4, 2},
+ {1, 5, 1, 13},
+ {1, 5, 2, 14},
+ {1, 5, 3, 15},
+ {1, 5, 4, 3},
+ {1, 6, 1, 22},
+ {1, 6, 2, 23},
+ {1, 6, 3, 24},
+ {1, 6, 4, 1}], ML_Core.Types.NumericField);
+
+ EXPORT expTrainData4 := DATASET([{1, 1, 1, 1},
+ {1, 1, 2, 2},
+ {1, 1, 3, 1},
+ {1, 1, 4, 3},
+ {1, 2, 1, 4},
+ {1, 2, 2, 5},
+ {1, 2, 3, 2},
+ {1, 2, 4, 6},
+ {1, 3, 1, 7},
+ {1, 3, 2, 8},
+ {1, 3, 3, 3},
+ {1, 3, 4, 9},
+ {1, 4, 1, 10},
+ {1, 4, 2, 11},
+ {1, 4, 3, 2},
+ {1, 4, 4, 12},
+ {1, 5, 1, 13},
+ {1, 5, 2, 14},
+ {1, 5, 3, 3},
+ {1, 5, 4, 15},
+ {1, 6, 1, 22},
+ {1, 6, 2, 23},
+ {1, 6, 3, 1},
+ {1, 6, 4, 24}], ML_Core.Types.NumericField);
+
+ EXPORT expTestData := DATASET([{1, 1, 1, 16},
+ {1, 1, 2, 17},
+ {1, 1, 3, 18},
+ {1, 1, 4, 2},
+ {1, 2, 1, 19},
+ {1, 2, 2, 20},
+ {1, 2, 3, 21},
+ {1, 2, 4, 3},
+ {1, 3, 1, 25},
+ {1, 3, 2, 26},
+ {1, 3, 3, 27},
+ {1, 3, 4, 2},
+ {1, 4, 1, 28},
+ {1, 4, 2, 29},
+ {1, 4, 3, 30},
+ {1, 4, 4, 1},
+ {1, 5, 1, 31},
+ {1, 5, 2, 32},
+ {1, 5, 3, 33},
+ {1, 5, 4, 1},
+ {1, 6, 1, 34},
+ {1, 6, 2, 35},
+ {1, 6, 3, 36},
+ {1, 6, 4, 3}], ML_Core.Types.NumericField);
+
+ EXPORT expTestData3 := DATASET([{1, 1, 1, 16},
+ {1, 1, 2, 17},
+ {1, 1, 3, 18},
+ {1, 1, 4, 2},
+ {1, 2, 1, 19},
+ {1, 2, 2, 20},
+ {1, 2, 3, 21},
+ {1, 2, 4, 3},
+ {1, 3, 1, 28},
+ {1, 3, 2, 29},
+ {1, 3, 3, 30},
+ {1, 3, 4, 1}], ML_Core.Types.NumericField);
+
+ EXPORT expTestData4 := DATASET([{1, 1, 1, 16},
+ {1, 1, 2, 17},
+ {1, 1, 3, 2},
+ {1, 1, 4, 18},
+ {1, 2, 1, 19},
+ {1, 2, 2, 20},
+ {1, 2, 3, 3},
+ {1, 2, 4, 21},
+ {1, 3, 1, 28},
+ {1, 3, 2, 29},
+ {1, 3, 3, 1},
+ {1, 3, 4, 30}], ML_Core.Types.NumericField);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/RunUtilsTests.ecl b/Preprocessing/Test/Functional/TestUtils/RunUtilsTests.ecl
new file mode 100644
index 0000000..521b481
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/RunUtilsTests.ecl
@@ -0,0 +1,15 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+/**
+ * Runs all utils tests
+ */
+$.TestDatasetComparator.RunDatasetComparatorTests;
+
+$.TestGetCategories.TestValidInput();
+
+$.TestGetFeatureNames.TestValidInput();
+
+$.TestLabelEncoder.TestMapAFeatureCategories.TestValidInput();
+$.TestLabelEncoder.TestMapCategoriesToValues.TestValidInput();
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/RunDatasetComparatorTests.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/RunDatasetComparatorTests.ecl
new file mode 100644
index 0000000..55a85ed
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/RunDatasetComparatorTests.ecl
@@ -0,0 +1,21 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+/**
+ * Runs all the dataset comparator tests
+ */
+$.TestGetFieldsInfo.TestSimpleRecord();
+$.TestGetFieldsInfo.TestComplexRecord();
+
+$.TestAreOfSameType.TestSameSimpleRecord();
+$.TestAreOfSameType.TestSameComplexRecord();
+$.TestAreOfSameType.TestDifferentRecord();
+
+$.TestGetRowValuesAndTypes.TestValidInput();
+
+$.TestAreEqualRows.TestEqualRows();
+$.TestAreEqualRows.TestDifferentRows();
+
+$.TestCompare.TestEqualData();
+$.TestCompare.TestRowDifference();
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreEqualRows.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreEqualRows.ecl
new file mode 100644
index 0000000..bb325d6
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreEqualRows.ecl
@@ -0,0 +1,30 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^.^ as ML_Core;
+DatasetComparator := ML_Core.Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test AreEqualRows function in DatasetComparator module.
+ */
+EXPORT TestAreEqualRows := MODULE
+ SHARED sample1 := $.TestDataAndTypes.sample1;
+ SHARED sample2 := $.TestDataAndTypes.sample2;
+
+ /**
+ * Testing with two equal rows
+ */
+ EXPORT TestEqualRows() := FUNCTION
+ result := DatasetComparator.AreEqualRows(sample1[1], sample1[1]);
+ RETURN ASSERT(result = TRUE, 'TestEqualRows Failed');
+ END;
+
+ /**
+ * Testing with different rows
+ */
+ EXPORT TestDifferentRows() := FUNCTION
+ result := DatasetComparator.AreEqualRows(sample1[1], sample2[1]);
+ RETURN ASSERT(result = FALSE, 'TestDifferentRows Failed');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreOfSameType.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreOfSameType.ecl
new file mode 100644
index 0000000..eecaf16
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestAreOfSameType.ecl
@@ -0,0 +1,44 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^.^ as ML_Core;
+DatasetComparator := ML_Core.Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test AreOfSameType function in DatasetComparator module.
+ */
+EXPORT TestAreOfSameType := MODULE
+ SHARED TestRecord1 := $.TestDataAndTypes.TestRecord1;
+ SHARED TestRecord2 := $.TestDataAndTypes.TestRecord2;
+
+ /**
+ * Testing with two datasets with same and simple record structure.
+ */
+ EXPORT TestSameSimpleRecord() := FUNCTION
+ dta1 := DATASET([{'a', 'b'}], TestRecord1);
+ dta2 := DATASET([{'c', 'd'}], TestRecord1);
+ result := DatasetComparator.AreOfSameType(dta1, dta2);
+ RETURN ASSERT(result = TRUE, 'TestSameSimpleRecord Failed');
+ END;
+
+ /**
+ * Testing with two datasets with same and complex record structure
+ */
+ EXPORT TestSameComplexRecord() := FUNCTION
+ dta1 := DATASET([{0, 'a', 'b', 0, 'c', {'d', 'e'}, [{'a', 'b'}]}], TestRecord2);
+ dta2 := DATASET([{1, 'c', 'd', 1, 'e', {'f', 'g'}, [{'a', 'b'}]}], TestRecord2);
+ result := DatasetComparator.AreOfSameType(dta1, dta2);
+ RETURN ASSERT(result = TRUE, 'TestSameComplexRecord Failed');
+ END;
+
+ /**
+ * Testing with two datasets different record structure
+ */
+ EXPORT TestDifferentRecord() := FUNCTION
+ dta1 := DATASET([{'a', 'b'}], TestRecord1);
+ dta2 := DATASET([{1, 'c', 'd', 1, 'e', {'f', 'g'}, [{'a', 'b'}]}], TestRecord2);
+ result := DatasetComparator.AreOfSameType(dta1, dta2);
+ RETURN ASSERT(result = FALSE, 'TestDifferentRecord Failed');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestCompare.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestCompare.ecl
new file mode 100644
index 0000000..4e3af2f
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestCompare.ecl
@@ -0,0 +1,41 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^.^ as ML_Core;
+DatasetComparator := ML_Core.Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test Compare function in DatasetComparator module.
+ */
+EXPORT TestCompare := MODULE
+ SHARED sample1 := $.TestDataAndTypes.sample1;
+ SHARED sample1_2 := $.TestDataAndTypes.sample1_2;
+ SHARED sample2 := $.TestDataAndTypes.sample2;
+ SHARED sample2_2 := $.TestDataAndTypes.sample2_2;
+
+ /**
+ * Testing with two equal datasets.
+ */
+ EXPORT TestEqualData() := FUNCTION
+ result := DatasetComparator.Compare(sample1, sample1);
+ RETURN ASSERT(result = 0, 'TestEqualData Failed');
+ END;
+
+ /**
+ * Testing with two datasets with same type but different number of rows.
+ */
+ EXPORT TestDifferentNumberOfRows() := FUNCTION
+ result := DatasetComparator.Compare(sample1, sample1_2);
+ RETURN ASSERT(result = -2, 'TestDifferentNumberOfRows Failed');
+ END;
+
+ /**
+ * Testing with two datasets with same number of rows and data structure
+ * but differing at a row.
+ */
+ EXPORT TestRowDifference() := FUNCTION
+ result := DatasetComparator.Compare(sample2, sample2_2);
+ RETURN ASSERT(result = 2, 'TestRowDifference Failed');
+ END;
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestDataAndTypes.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestDataAndTypes.ecl
new file mode 100644
index 0000000..5587b53
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestDataAndTypes.ecl
@@ -0,0 +1,45 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+/**
+ * Data and Record structures used by TestDatasetComparator Modules
+ */
+EXPORT TestDataAndTypes := MODULE
+ //test record structures
+ EXPORT TestRecord1 := RECORD
+ STRING10 field1;
+ STRING20 field2;
+ END;
+
+ EXPORT TestRecord2 := RECORD
+ UNSIGNED4 field1;
+ STRING10 field2;
+ string field3;
+ UNSIGNED1 field4;
+ STRING20 field5;
+ TestRecord1 field6;
+ DATASET(TestRecord1) field7;
+ END;
+
+ EXPORT TestRecord3 := RECORD
+ BOOLEAN field1;
+ UNSIGNED4 field2;
+ INTEGER field3;
+ DECIMAL field4;
+ REAL4 field5;
+ STRING10 field6;
+ QSTRING field7;
+ END;
+
+ //sample data for testing
+ EXPORT sample1 := DATASET([{True, 5, -8, 2.5, 2345.78238, 'sample1', '1'},
+ {True, 5, -8, 2.5, 2345.78238, 'sample1', '1'}], TestRecord3);
+ EXPORT sample1_2 := DATASET([{True, 5, -8, 2.5, 2345.78238, 'sample1_2', '1_2'},
+ {True, 15, -18, 12.5, 23, 'sample1_2', '1_2'}], TestRecord3);
+
+ EXPORT sample2 := DATASET([{False, 450, 1000, -21212.31, 9852.191, 'sample2', '2'},
+ {True, 450, 1000, -21212.31, 9852.191, 'sample2', '2'}], TestRecord3);
+ EXPORT sample2_2 := DATASET([{False, 450, 1000, -21212.31, 9852.191, 'sample2', '2'},
+ {False, 450, 1000, -21212.31, 9852.191, 'sample2_2', '2_2'}], TestRecord3);
+END;
\ No newline at end of file
diff --git a/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestGetFieldsInfo.ecl b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestGetFieldsInfo.ecl
new file mode 100644
index 0000000..85b060a
--- /dev/null
+++ b/Preprocessing/Test/Functional/TestUtils/TestDatasetComparator/TestGetFieldsInfo.ecl
@@ -0,0 +1,49 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. All rights reserved.
+############################################################################## */
+
+IMPORT $.^.^.^.^.^ as ML_Core;
+DatasetComparator := ML_Core.Preprocessing.Utils.DatasetComparator;
+
+/**
+ * Test GetFieldsInfo function in DatasetComparator module.
+ */
+EXPORT TestGetFieldsInfo := MODULE
+ SHARED FieldInfo := DatasetComparator.Types.FieldInfo;
+ SHARED TestRecord1 := $.TestDataAndTypes.TestRecord1;
+ SHARED TestRecord2 := $.TestDataAndTypes.TestRecord2;
+
+ /**
+ * Test getting fields' info from a simple record structure.
+ */
+ EXPORT TestSimpleRecord() := FUNCTION
+ testData := DATASET([{'a', 'b'}], TestRecord1);
+ result := DatasetComparator.GetFieldsInfo(testData);
+ expected := DATASET([{'0', 'field1', 'string', '10'}, {'1', 'field2', 'string', '20'}], FieldInfo);
+ bothResults := MERGE(SORT(result, position), SORT(expected, position), SORTED(position));
+ deduped := DEDUP(bothResults);
+ RETURN ASSERT(COUNT(deduped) = COUNT(expected), 'TestSimpleRecord Failed');
+ END;
+
+ /**
+ * Test getting fields' info from a complex record structure.
+ */
+ EXPORT TestComplexRecord() := FUNCTION
+ testData := DATASET([{0, 'a', 'b', 0, 'c', {'d', 'e'}, [{'a', 'b'}]}], TestRecord2);
+ result := DatasetComparator.GetFieldsInfo(testData);
+ expected := DATASET([{' 0', 'field1', 'unsigned', '4'},
+ {' 1', 'field2', 'string', '10'},
+ {' 2', 'field3', 'string', '-15'},
+ {' 3', 'field4', 'unsigned', '1'},
+ {' 4', 'field5', 'string', '20'},
+ {' 5', 'field6', 'testrecord1', '30'},
+ {' 6', 'field1', 'string', '10'},
+ {' 7', 'field2', 'string', '20'},
+ {' 8', 'field7', 'table of