diff --git a/docker-compose.yml b/docker-compose.yml index fc3352f..4ddfced 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: volumes: - ./:/app working_dir: /app - command: ["sh", "install_compile.sh"] + command: ["sh", "install_compile.sh", "12"] x86_64_builder: image: gcc:latest @@ -14,4 +14,4 @@ services: volumes: - ./:/app working_dir: /app - command: ["sh", "install_compile.sh"] + command: ["sh", "install_compile.sh", "12"] diff --git a/install_compile.sh b/install_compile.sh index 2d07275..9add927 100644 --- a/install_compile.sh +++ b/install_compile.sh @@ -1,4 +1,27 @@ -apt-get update && apt-get install -y python3 curl cmake python3-dev +#!/usr/bin/env bash +set -euo pipefail + +# Simple installer: pass a suffix (e.g. 12 or 13) to install python3.X +# Usage: ./install_compile.sh [12] + +SUFFIX="${1:-}" + +apt-get update +apt-get install -y --no-install-recommends curl cmake build-essential + +if [ -n "$SUFFIX" ]; then + PYPKG="python3.$SUFFIX" + PYDEV="python3.$SUFFIX-dev" + echo "Installing ${PYPKG} and ${PYDEV}" + apt-get install -y --no-install-recommends ${PYPKG} ${PYDEV} || ( + echo "Failed to install ${PYPKG}. Falling back to system python3." && apt-get install -y --no-install-recommends python3 python3-dev + ) +else + echo "No suffix provided. Installing system python3 and headers." + apt-get install -y --no-install-recommends python3 python3-dev +fi + curl -LsSf https://astral.sh/uv/install.sh | sh + sh compile.sh diff --git a/src/detectmateperformance/_core/_type/element.cpp b/src/detectmateperformance/_core/_type/element.cpp index 6dc5a4a..8eaa6b6 100644 --- a/src/detectmateperformance/_core/_type/element.cpp +++ b/src/detectmateperformance/_core/_type/element.cpp @@ -3,14 +3,11 @@ std::string postProcessTemp(const std::string& input){ - std::string pattern = "\\bVAR\\b"; - std::regex re(pattern); - std::string result = std::regex_replace(input, re, "<*>"); - return result; + return input; // For now it will be save like this } -std::deque postProcessVars(const std::string& input_vars) { - return preprocessing(input_vars); +std::string postProcessVars(const std::string& input_vars) { + return input_vars; // For now it will be save like this } ParsedElement::ParsedElement( diff --git a/src/detectmateperformance/_core/_type/element.h b/src/detectmateperformance/_core/_type/element.h index 574cf4a..c0713a3 100644 --- a/src/detectmateperformance/_core/_type/element.h +++ b/src/detectmateperformance/_core/_type/element.h @@ -11,14 +11,14 @@ std::string postProcessTemp(const std::string& input); -std::deque postProcessVars(const std::string& input_vars); +std::string postProcessVars(const std::string& input_vars); class ParsedElement { public: int event_id; std::string log_template; - std::deque variables; + std::string variables; ParsedElement( int event_id, std::string log_template, std::string variables diff --git a/src/detectmateperformance/_core/_type/parsed.cpp b/src/detectmateperformance/_core/_type/parsed.cpp index 55e1dba..27c85a8 100644 --- a/src/detectmateperformance/_core/_type/parsed.cpp +++ b/src/detectmateperformance/_core/_type/parsed.cpp @@ -88,8 +88,8 @@ std::vector ParsedMessages::getAllElemts() { return templates; } -std::vector> ParsedMessages::getAllVar() { - std::vector> vars(this->size()); +std::vector ParsedMessages::getAllVar() { + std::vector vars(this->size()); for (int i = 0; i < this->size(); i++) { vars[i] = this->getElemWithVar(i).variables; diff --git a/src/detectmateperformance/_core/_type/parsed.h b/src/detectmateperformance/_core/_type/parsed.h index ceb6f9b..b6b721e 100644 --- a/src/detectmateperformance/_core/_type/parsed.h +++ b/src/detectmateperformance/_core/_type/parsed.h @@ -35,7 +35,7 @@ class ParsedMessages { std::vector getAllIDs(); std::vector getAllElemts(); - std::vector> getAllVar(); + std::vector getAllVar(); int size(); std::pair shape(); diff --git a/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so b/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so index ff43b91..e4bf88d 100755 Binary files a/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so and b/src/detectmateperformance/lib/bind_class.cpython-312-x86_64-linux-gnu.so differ diff --git a/src/detectmateperformance/lib/bind_class.cpython-313-aarch64-linux-gnu.so b/src/detectmateperformance/lib/bind_class.cpython-313-aarch64-linux-gnu.so index d637585..019e235 100755 Binary files a/src/detectmateperformance/lib/bind_class.cpython-313-aarch64-linux-gnu.so and b/src/detectmateperformance/lib/bind_class.cpython-313-aarch64-linux-gnu.so differ diff --git a/src/detectmateperformance/lib/bind_class.cpython-313-x86_64-linux-gnu.so b/src/detectmateperformance/lib/bind_class.cpython-313-x86_64-linux-gnu.so index cc093fe..7bcdfd2 100755 Binary files a/src/detectmateperformance/lib/bind_class.cpython-313-x86_64-linux-gnu.so and b/src/detectmateperformance/lib/bind_class.cpython-313-x86_64-linux-gnu.so differ diff --git a/src/detectmateperformance/pipeline_op.py b/src/detectmateperformance/pipeline_op.py index e525dd0..1626358 100644 --- a/src/detectmateperformance/pipeline_op.py +++ b/src/detectmateperformance/pipeline_op.py @@ -41,6 +41,13 @@ def add_parsed(df: pl.DataFrame, results: ParsedLogs) -> pl.DataFrame: return df +def postprocessing(df: pl.DataFrame) -> pl.DataFrame: + df = df.with_columns(pl.col("Templates").str.replace_all("VAR", "<*>")) + if "ParamList" in df.columns: + df = df.with_columns(pl.col("ParamList").str.split(by=" ")) + return df + + def run_batches( func: Callable[[list[str], bool, int], ParsedLogs], table: pl.DataFrame, @@ -62,7 +69,8 @@ def run_batches( df = pl.concat([df, add_parsed(df=table[i-batch: i], results=results)]) del results - return df + print(">>> Postprocessing results") + return postprocessing(df) def run_full_pipeline( diff --git a/src/detectmateperformance/types_.py b/src/detectmateperformance/types_.py index 569dfd8..7a2adf2 100644 --- a/src/detectmateperformance/types_.py +++ b/src/detectmateperformance/types_.py @@ -48,7 +48,7 @@ class ParsedLogElement: def __init__(self, parsed_ele: ParsedElement) -> None: self.event_id: int = parsed_ele.event_id self.log_template: str = parsed_ele.log_template - self.variables: list[str] = parsed_ele.variables + self.variables: str = parsed_ele.variables def to_dict(self) -> dict[str, Any]: return { @@ -57,6 +57,13 @@ def to_dict(self) -> dict[str, Any]: "ParamList": self.variables, } + def postprocess_to_dict(self) -> dict[str, Any]: + return { + "EventID": self.event_id, + "Template": self.log_template.replace("VAR", "<*>"), + "ParamList": self.variables.split(" ") if self.variables != "" else [], + } + class ParsedLogs: def __init__( @@ -85,7 +92,7 @@ def __getitem__(self, idx: int) -> dict[str, Any]: result = ParsedLogElement(self.inst.get_elem_with_var(idx)) else: result = ParsedLogElement(self.inst.get_elem(idx)) - return result.to_dict() + return result.postprocess_to_dict() def __setitem__(self, idx: int, values: str | tuple[str, str]) -> str: if self.with_vars: diff --git a/tests/test_c/test_message.cpp b/tests/test_c/test_message.cpp index b40479f..4ca6d7a 100644 --- a/tests/test_c/test_message.cpp +++ b/tests/test_c/test_message.cpp @@ -233,7 +233,7 @@ TEST(TreeMatchTest, MatchString) { ParsedMessages* result2 = matcher->match_string("hi general mr. and mrs. kenobi"); EXPECT_EQ(result2->size(), 1); - EXPECT_EQ(result2->getElem(0).log_template, "hi general <*> kenobi"); + EXPECT_EQ(result2->getElem(0).log_template, "hi general VAR kenobi"); ParsedMessages* result3 = matcher->match_string("hi random guy"); EXPECT_EQ(result3->size(), 1); @@ -250,15 +250,15 @@ TEST(TreeMatchTest, MatchStringWithVar) { MatchTree* matcher = new MatchTree(temp); ParsedElement result1 = matcher->match_string_with_var("hi there")->getElemWithVar(0); - std::deque expected1 = {}; + std::string expected1 = ""; EXPECT_EQ(result1.log_template, "hi there"); EXPECT_EQ(result1.variables, expected1); ParsedElement result2 = matcher->match_string_with_var( "hi general mr. and mrs. kenobi" )->getElemWithVar(0); - std::deque expected2 = {"mr", "and", "mrs"}; - EXPECT_EQ(result2.log_template, "hi general <*> kenobi"); + std::string expected2 = "mr and mrs"; + EXPECT_EQ(result2.log_template, "hi general VAR kenobi"); EXPECT_EQ(result2.variables, expected2); ParsedElement result3 = matcher->match_string_with_var( @@ -269,8 +269,8 @@ TEST(TreeMatchTest, MatchStringWithVar) { ParsedElement result4 = matcher->match_string_with_var( "load 1213 asd from 112 bye" )->getElemWithVar(0); - std::deque expected4 = {"1213", "asd", "112", "bye"}; - EXPECT_EQ(result4.log_template, "load <*> from <*>"); + std::string expected4 = "1213 asd 112 bye"; + EXPECT_EQ(result4.log_template, "load VAR from VAR"); EXPECT_EQ(result4.variables, expected4); delete matcher; @@ -321,12 +321,12 @@ TEST(TreeMatchTest, MatchStringBatchVar) { std::vector msg_ex = { "hi there", "hi general <*> kenobi", "template not found", "load <*> from <*>" }; + std::vector ex_msg_ex = { + "hi there", "hi general VAR kenobi", "template not found", "load VAR from VAR" + }; std::vector vector_vars = { "", "mr and mrs", "", "1213 asd 112 bye" }; - std::vector> ex_vector_vars = { - {}, {"mr", "and", "mrs"}, {}, {"1213", "asd", "112", "bye"} - }; Templates* temp2 = new Templates(sequences); ParsedMessages* expected = new ParsedMessages(temp2, 4); @@ -347,8 +347,8 @@ TEST(TreeMatchTest, MatchStringBatchVar) { if (msg_ex[i] == "template not found") EXPECT_EQ(-1, aux.event_id); - EXPECT_EQ(msg_ex[i], aux.log_template); - EXPECT_EQ(aux.variables, ex_vector_vars[i]); + EXPECT_EQ(ex_msg_ex[i], aux.log_template); + EXPECT_EQ(aux.variables, vector_vars[i]); } delete matcher; @@ -367,7 +367,7 @@ TEST(ParsedMessagesTest, HardCases1) { ParsedMessages* result1 = matcher->match_string(log); EXPECT_EQ(result1->size(), 1); - EXPECT_EQ(result1->getElem(0).log_template, "<*> floating point alignment exceptions"); + EXPECT_EQ(result1->getElem(0).log_template, "VAR floating point alignment exceptions"); } @@ -383,7 +383,7 @@ TEST(ParsedMessagesTest, HardCases2) { ParsedMessages* result1 = matcher->match_string(log); EXPECT_EQ(result1->size(), 1); - EXPECT_EQ(result1->getElem(0).log_template, "<*> Exception writing block <*> to mirror <*>"); + EXPECT_EQ(result1->getElem(0).log_template, "VAR Exception writing block VAR to mirror VAR"); } @@ -399,7 +399,7 @@ TEST(ParsedMessagesTest, HardCases3) { ParsedMessages* result1 = matcher->match_string(log); EXPECT_EQ(result1->size(), 1); - EXPECT_EQ(result1->getElem(0).log_template, "rts kernel terminated for reason <*>"); + EXPECT_EQ(result1->getElem(0).log_template, "rts kernel terminated for reason VAR"); } diff --git a/tests/test_c/test_type.cpp b/tests/test_c/test_type.cpp index a9292e6..9a1eecc 100644 --- a/tests/test_c/test_type.cpp +++ b/tests/test_c/test_type.cpp @@ -48,25 +48,25 @@ TEST(ParsedMessagesTest, ParsedElementtest) { ParsedElement* parsed = new ParsedElement( 0, "VAR template VAR VAR", "var1 var2 var3" ); - std::deque expected = {"var1", "var2", "var3"}; + std::string expected = "var1 var2 var3"; EXPECT_EQ(parsed->event_id, 0); - EXPECT_EQ(parsed->log_template, "<*> template <*> <*>"); + EXPECT_EQ(parsed->log_template, "VAR template VAR VAR"); EXPECT_EQ(parsed->variables, expected); ParsedElement* parsed2 = new ParsedElement( 0, "VAR template VAR VAR" ); - std::deque expected2 = {}; + std::string expected2 = ""; EXPECT_EQ(parsed2->event_id, 0); - EXPECT_EQ(parsed2->log_template, "<*> template <*> <*>"); + EXPECT_EQ(parsed2->log_template, "VAR template VAR VAR"); EXPECT_EQ(parsed2->variables, expected2); ParsedElement* parsed3 = new ParsedElement( -1, "template not found", "5 1 2" ); - std::deque expected3 = {"5", "1", "2"}; + std::string expected3 = "5 1 2"; EXPECT_EQ(parsed3->event_id, -1); EXPECT_EQ(parsed3->log_template, "template not found"); @@ -75,7 +75,7 @@ TEST(ParsedMessagesTest, ParsedElementtest) { ParsedElement* parsed4 = new ParsedElement( -1, "template not found" ); - std::deque expected4 = {}; + std::string expected4 = ""; EXPECT_EQ(parsed4->event_id, -1); EXPECT_EQ(parsed4->log_template, "template not found"); @@ -107,8 +107,8 @@ TEST(ParsedMessagesTest, GetNext) { parsed.setElem(2, "Goodbye VAR"); parsed.setElem(3, "random thing"); - std::string temp1 = "Hello <*> world <*>"; - std::string temp2 = "Goodbye <*>"; + std::string temp1 = "Hello VAR world VAR"; + std::string temp2 = "Goodbye VAR"; std::string temp3 = "template not found"; EXPECT_EQ(parsed.getElem(0).log_template, temp1); @@ -155,11 +155,11 @@ TEST(ParsedMessagesTest, GetNextWithVar) { parsed.setElemWithVar(2, "Goodbye VAR", var2); parsed.setElemWithVar(3, "random thing", var1); - std::string temp1 = "Hello <*> world <*>"; - std::string temp2 = "Goodbye <*>"; + std::string temp1 = "Hello VAR world VAR"; + std::string temp2 = "Goodbye VAR"; std::string temp3 = "template not found"; - std::deque evar1 = {}; - std::deque evar2 = {"a", "b"}; + std::string evar1 = ""; + std::string evar2 = "a b"; EXPECT_EQ(parsed.getElemWithVar(0).log_template, temp1); EXPECT_EQ(parsed.getElemWithVar(0).variables, evar1); @@ -184,7 +184,7 @@ TEST(ParsedMessagesTest, GetAllVar) { parsed.setElemWithVar(2, "Goodbye VAR", var2); parsed.setElemWithVar(3, "random thing", var1); - std::vector> result = parsed.getAllVar(); + std::vector result = parsed.getAllVar(); for (int i = 0; i < result.size(); i++) { EXPECT_EQ(parsed.getElemWithVar(i).variables, result[i]); diff --git a/tests/test_python/test_types.py b/tests/test_python/test_types.py index 9367eb6..300c2c8 100644 --- a/tests/test_python/test_types.py +++ b/tests/test_python/test_types.py @@ -64,7 +64,7 @@ def test_get_all_templates(self): parsed[0] = "Hello VAR world VAR" parsed[1] = "ciaoo bellaaa" - expected = ["Hello <*> world <*>", "template not found"] + expected = ["Hello VAR world VAR", "template not found"] assert parsed.get_all_templates() == expected def test_add_elements_with_vars(self): @@ -83,7 +83,7 @@ def test_get_all_variables(self): parsed[0] = ("Hello VAR world VAR", "") parsed[1] = ("ciaoo bellaaa", "a b c") - expected = [[], ["a", "b", "c"]] + expected = ["", "a b c"] assert parsed.get_all_vars() == expected def test_add_elements_shape_(self):