Imp/tparsed

CMakeLists.txt

@@ -12,7 +12,6 @@ FetchContent_Declare(
   URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
 )
 
-# Current not working, figure it out why! TODO: Fix pybind11
 FetchContent_Declare(
   pybind11
   GIT_REPOSITORY https://github.com/pybind/pybind11.git
@@ -44,6 +43,12 @@ add_library(
   src/detectmateperformance/_core/_type/templates.cpp
 )
 
+add_library(
+  parsedelement
+  src/detectmateperformance/_core/_type/element.h
+  src/detectmateperformance/_core/_type/element.cpp
+)
+
 add_library(
   parsed
   src/detectmateperformance/_core/_type/parsed.h
@@ -112,6 +117,8 @@ pybind11_add_module(
   src/detectmateperformance/_core/aux.cpp
   src/detectmateperformance/_core/_type/templates.h
   src/detectmateperformance/_core/_type/templates.cpp
+  src/detectmateperformance/_core/_type/element.h
+  src/detectmateperformance/_core/_type/element.cpp
   src/detectmateperformance/_core/_type/parsed.h
   src/detectmateperformance/_core/_type/parsed.cpp
   src/detectmateperformance/_core/template_matcher/tree.h
@@ -128,6 +135,7 @@ target_link_libraries(
   test_c
   GTest::gtest_main
   templates
+  parsedelement
   parsed
   tree
   variable
@@ -140,6 +148,7 @@ target_link_libraries(
   test_type
   GTest::gtest_main
   templates
+  parsedelement
   parsed
   aux
 )
@@ -154,6 +163,7 @@ target_link_libraries(
   test_matcher_vars
   GTest::gtest_main
   templates
+  parsedelement
   parsed
   tree
   variable

compile.sh

@@ -16,6 +16,7 @@ cd ..
 mkdir lib
 cp build/bind_class* src/detectmateperformance/lib/
 chmod 775 src/detectmateperformance/lib/*
+chmod 775 -R build/*
 
 # Install package
 $HOME/.local/bin/uv pip uninstall detectmateperformance

docker-compose.yml

@@ -6,20 +6,12 @@ services:
     volumes:
       - ./:/app
     working_dir: /app
-    command: >
-      sh -c "
-      apt-get update && apt-get install -y python3 curl cmake python3-dev &&
-      curl -LsSf https://astral.sh/uv/install.sh | sh &&
-      sh compile.sh"
+    command: ["sh", "install_compile.sh"]
 
   x86_64_builder:
     image: gcc:latest
     container_name: x86_64_builder
     volumes:
       - ./:/app
     working_dir: /app
-    command: >
-      sh -c "
-      apt-get update && apt-get install -y python3 curl cmake python3-dev &&
-      curl -LsSf https://astral.sh/uv/install.sh | sh &&
-      sh compile.sh"
+    command: ["sh", "install_compile.sh"]

install_compile.sh

@@ -0,0 +1,4 @@
+
+apt-get update && apt-get install -y python3 curl cmake python3-dev
+curl -LsSf https://astral.sh/uv/install.sh | sh
+sh compile.sh

src/detectmateperformance/_core/_type/element.cpp

@@ -0,0 +1,34 @@
+
+#include "element.h"
+
+
+std::string postProcessTemp(const std::string& input){
+    std::string pattern = "\\bVAR\\b";
+    std::regex re(pattern);
+    std::string result = std::regex_replace(input, re, "<*>");
+    return result;
+}
+
+std::deque<std::string> postProcessVars(const std::string& input_vars) {
+    return preprocessing(input_vars);
+}
+
+ParsedElement::ParsedElement(
+    int event_id, std::string log_template, std::string variables
+) {
+    this->event_id = event_id;
+    this->log_template = postProcessTemp(log_template);
+    this->variables = postProcessVars(variables);
+}
+
+ParsedElement::ParsedElement(
+    int event_id, std::string log_template
+) {
+    this->event_id = event_id;
+    this->log_template = postProcessTemp(log_template);
+    this->variables = {};
+}
+
+ParsedElement::~ParsedElement() {
+
+}

src/detectmateperformance/_core/_type/element.h

@@ -0,0 +1,32 @@
+#ifndef M_PARSEDELEM_H
+#define M_PARSEDELEM_H
+
+#include <string>
+#include <deque>
+#include "../aux.h"
+#include <regex>
+
+
+
+
+std::string postProcessTemp(const std::string& input);
+
+std::deque<std::string> postProcessVars(const std::string& input_vars);
+
+class ParsedElement {
+
+public:
+    int event_id;
+    std::string log_template;
+    std::deque<std::string> variables;
+
+    ParsedElement(
+        int event_id, std::string log_template, std::string variables
+    );
+    ParsedElement(int event_id, std::string log_template);
+
+    ~ParsedElement();
+
+};
+
+#endif

src/detectmateperformance/_core/_type/parsed.cpp

@@ -29,21 +29,26 @@ int ParsedMessages::getElemID(int n) {
     return this->event_ids[n];
 }
 
-std::string ParsedMessages::getElem(int n) {
+ParsedElement ParsedMessages::getElem(int n) {
 
     int event_idsf = this->getElemID(n);
     if (event_idsf == -1) {
-        return "template not found";
+        return ParsedElement(-1, "template not found");
     }
 
-    return id_to_template[event_idsf];
+    return ParsedElement(event_idsf, id_to_template[event_idsf]);
 }
 
-std::pair<std::string, std::string> ParsedMessages::getElemWithVar(int n) {
-    std::string temp = ParsedMessages::getElem(n);
-    std::string vars = this->variables[n];
+ParsedElement ParsedMessages::getElemWithVar(int n) {
+    std::string variables = this->variables[n];
+    int event_idsf = this->getElemID(n);
+
+    if (event_idsf == -1) {
+        return ParsedElement(-1, "template not found", variables);
+    }
+    std::string template_ = id_to_template[event_idsf];
 
-    return std::make_pair(temp, vars);
+    return ParsedElement(event_idsf, template_, variables);
 }
 
 void ParsedMessages::setElem(int n, std::string template_) {
@@ -77,14 +82,20 @@ std::vector<std::string> ParsedMessages::getAllElemts() {
     std::vector<std::string> templates(this->size());
 
     for (int i = 0; i < this->size(); i++) {
-        templates[i] = this->getElem(i);
+        templates[i] = this->getElem(i).log_template;
     }
 
     return templates;
 }
 
-std::vector<std::string> ParsedMessages::getAllVar() {
-    return this->variables;
+std::vector<std::deque<std::string>> ParsedMessages::getAllVar() {
+    std::vector<std::deque<std::string>> vars(this->size());
+
+    for (int i = 0; i < this->size(); i++) {
+        vars[i] = this->getElemWithVar(i).variables;
+    }
+
+    return vars;
 }
 
 int ParsedMessages::size() {

src/detectmateperformance/_core/_type/parsed.h

@@ -7,6 +7,7 @@
 #include <deque>
 
 #include "templates.h"
+#include "element.h"
 
 #include <functional>
 
@@ -25,16 +26,16 @@ class ParsedMessages {
     ~ParsedMessages();
 
     int getElemID(int n);
-    std::string getElem(int n);
-    std::pair<std::string, std::string> getElemWithVar(int n);
+    ParsedElement getElem(int n);
+    ParsedElement getElemWithVar(int n);
     void setElem(int n, std::string template_);
     void setElemWithVar(
         int n, std::string template_, std::string vars
     );
 
     std::vector<int> getAllIDs();
     std::vector<std::string> getAllElemts();
-    std::vector<std::string> getAllVar();
+    std::vector<std::deque<std::string>> getAllVar();
 
     int size();
     std::pair<int, int> shape();

src/detectmateperformance/_core/bind_class.cpp

@@ -2,6 +2,7 @@
 #include <pybind11/stl.h>
 
 #include "template_matcher/match_tree.h"
+#include "_type/element.h"
 #include "_type/templates.h"
 #include "_type/parsed.h"
 
@@ -15,6 +16,10 @@ PYBIND11_MODULE(bind_class, m) {
         .def("get_next_template", &Templates::getNext)
         .def("size", &Templates::size)
         .def("shape", &Templates::shape);
+    py::class_<ParsedElement>(m, "ParsedElement")
+        .def_readonly("event_id", &ParsedElement::event_id)
+        .def_readonly("log_template", &ParsedElement::log_template)
+        .def_readonly("variables", &ParsedElement::variables);
     py::class_<ParsedMessages>(m, "Parsed")
         .def(py::init<Templates*, int>())
         .def("get_elem_id", &ParsedMessages::getElemID)

src/detectmateperformance/pipeline_op.py

@@ -41,13 +41,6 @@ def add_parsed(df: pl.DataFrame, results: ParsedLogs) -> pl.DataFrame:
     return df
 
 
-def postprocessing(df: pl.DataFrame) -> pl.DataFrame:
-    df = df.with_columns(pl.col("Templates").str.replace_all("VAR", "<*>"))
-    if "ParamList" in df.columns:
-        df = df.with_columns(pl.col("ParamList").str.split(by=" "))
-    return df
-
-
 def run_batches(
     func: Callable[[list[str], bool, int], ParsedLogs],
     table: pl.DataFrame,
@@ -69,8 +62,7 @@ def run_batches(
             df = pl.concat([df, add_parsed(df=table[i-batch: i], results=results)])
         del results
 
-    print(">>> Postprocessing results")
-    return postprocessing(df)
+    return df
 
 
 def run_full_pipeline(

src/detectmateperformance/types_.py

@@ -1,4 +1,6 @@
-from detectmateperformance.lib.bind_class import Templates, Parsed
+from detectmateperformance.lib.bind_class import Templates, Parsed, ParsedElement
+
+from typing import Any
 
 
 def _load_file(path: str) -> list[str]:
@@ -42,6 +44,20 @@ def from_file(cls, path: str) -> "LogTemplates":
         return cls(_load_file(path))
 
 
+class ParsedLogElement:
+    def __init__(self, parsed_ele: ParsedElement) -> None:
+        self.event_id: int = parsed_ele.event_id
+        self.log_template: str = parsed_ele.log_template
+        self.variables: list[str] = parsed_ele.variables
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "EventID": self.event_id,
+            "Template": self.log_template,
+            "ParamList": self.variables,
+        }
+
+
 class ParsedLogs:
     def __init__(
         self, templates: LogTemplates, n: int, with_vars: bool = False
@@ -64,10 +80,12 @@ def shape(self) -> tuple[int, int]:
     def __str__(self) -> str:
         return f"ParsedLogs(shape={self.shape()}, vars={self.with_vars})"
 
-    def __getitem__(self, idx: int) -> str | tuple[str, str]:
+    def __getitem__(self, idx: int) -> dict[str, Any]:
         if self.with_vars:
-            return self.inst.get_elem_with_var(idx)  # type: ignore
-        return self.inst.get_elem(idx)  # type: ignore
+            result = ParsedLogElement(self.inst.get_elem_with_var(idx))
+        else:
+            result = ParsedLogElement(self.inst.get_elem(idx))
+        return result.to_dict()
 
     def __setitem__(self, idx: int, values: str | tuple[str, str]) -> str:
         if self.with_vars:
@@ -80,7 +98,7 @@ def get_all_events_ids(self) -> list[int]:
     def get_all_templates(self) -> list[str]:
         return self.inst.get_all_elem()  # type: ignore
 
-    def get_all_vars(self) -> list[str] | None:
+    def get_all_vars(self) -> list[list[str]] | None:
         if self.with_vars:
             return self.inst.get_all_var()  # type: ignore
         return None