Add csv query reports to anomaly detection

Author: JohT

domains/anomaly-detection/anomalyDetectionPython.sh

@@ -162,14 +162,23 @@ anomaly_detection_labels() {
 
     local language
     language=$( extractQueryParameter "projection_language" "${@}" )
-    
+
     echo "anomalyDetectionPython: $(date +'%Y-%m-%dT%H:%M:%S%z') Labelling ${language} ${nodeLabel} anomalies..."
+    
+    # Within the absolute (full) report directory for anomaly detection, create a sub directory for every detailed type (Java_Package, Java_Type,...)
+    local detail_report_directory="${FULL_REPORT_DIRECTORY}/${language}_${nodeLabel}"
+    mkdir -p "${detail_report_directory}"
+
     execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeRemoveLabels.cypher" "${@}"
-    execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeAuthority.cypher" "${@}"
-    execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeBottleneck.cypher" "${@}"
-    execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeHub.cypher" "${@}"
-    execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeBridge.cypher" "${@}"
-    execute_cypher_summarized "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeOutlier.cypher" "${@}"
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeAuthority.cypher" "${@}" > "${detail_report_directory}/AnomalyArchetypeTopAuthority.csv"
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeBottleneck.cypher" "${@}" > "${detail_report_directory}/AnomalyArchetypeTopBottleneck.csv"
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeHub.cypher" "${@}" > "${detail_report_directory}/AnomalyArchetypeTopHub.csv"
+    # The following two label types require Python scripts to run first.
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeBridge.cypher" "${@}" > "${detail_report_directory}/AnomalyArchetypeTopBridge.csv"
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionArchetypeOutlier.cypher" "${@}" > "${detail_report_directory}/AnomalyArchetypeTopOutlier.csv"
+    # Output the top anomalies and their archetype + rank
+    execute_cypher "${ANOMALY_DETECTION_LABEL_CYPHER_DIR}/AnomalyDetectionTopAnomalies.cypher" "${@}" > "${detail_report_directory}/TopAnomalies.csv"
+
 }
 
 # Run the anomaly detection pipeline.

domains/anomaly-detection/labels/AnomalyDetectionTopAnomalies.cypher

@@ -0,0 +1,28 @@
+// List top anomalies
+
+   MATCH (codeUnit)
+   WHERE $projection_node_label IN labels(codeUnit)
+     AND codeUnit.anomalyScore > 0
+     AND codeUnit.anomalyLabel = 1
+   ORDER BY codeUnit.anomalyScore DESC
+   LIMIT 50
+OPTIONAL MATCH (artifact:Java:Artifact)-[:CONTAINS]->(codeUnit)
+    WITH *, artifact.name AS artifactName
+OPTIONAL MATCH (projectRoot:Directory)<-[:HAS_ROOT]-(proj:TS:Project)-[:CONTAINS]->(codeUnit)
+    WITH *, last(split(projectRoot.absoluteFileName, '/')) AS projectName
+    WITH *, coalesce(artifactName, projectName)            AS projectName
+  RETURN projectName
+        ,codeUnit.name                                     AS shortCodeUnitName
+        ,coalesce(codeUnit.fqn, codeUnit.globalFqn, codeUnit.fileName, codeUnit.signature, codeUnit.name) AS codeUnitName
+        ,codeUnit.anomalyScore                             AS anomalyScore
+        ,coalesce(codeUnit.anomalyAuthorityRank, 0)        AS authorityRank
+        ,coalesce(codeUnit.anomalyBottleneckRank, 0)       AS bottleneckRank
+        ,coalesce(codeUnit.anomalyBridgeRank, 0)           AS bridgeRank
+        ,coalesce(codeUnit.anomalyHubRank, 0)              AS hubRank
+        ,coalesce(codeUnit.anomalyOutlierRank, 0)          AS outlierRank
+        ,codeUnit.anomalyTopFeature1                       AS topFeature1
+        ,codeUnit.anomalyTopFeature2                       AS topFeature2
+        ,codeUnit.anomalyTopFeature3                       AS topFeature3
+        ,codeUnit.anomalyTopFeatureSHAPValue1              AS topFeature1Score
+        ,codeUnit.anomalyTopFeatureSHAPValue2              AS topFeature2Score
+        ,codeUnit.anomalyTopFeatureSHAPValue3              AS topFeature3Score