From 5a0bd4cf9cb56d63fdc9cce2185b54b064a095d2 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 11:55:26 -0300 Subject: [PATCH 01/13] make identation its own file --- spice/analyze.py | 2 +- spice/analyzers/__init__.py | 0 spice/{ => analyzers}/identation.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 spice/analyzers/__init__.py rename spice/{ => analyzers}/identation.py (100%) diff --git a/spice/analyze.py b/spice/analyze.py index 5d5f748..8ba9f20 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -11,7 +11,7 @@ # gustavo testando alguma coisa -from spice.identation import detect_indentation +from spice.analyzers.identation import detect_indentation # this will read the file extension and return the correct lexer def get_lexer_for_file(file_path): diff --git a/spice/analyzers/__init__.py b/spice/analyzers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spice/identation.py b/spice/analyzers/identation.py similarity index 100% rename from spice/identation.py rename to spice/analyzers/identation.py From 89c98d9eb296852fae40c7d08ca443d4da9e8fda Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 11:58:19 -0300 Subject: [PATCH 02/13] separate count lines into own file --- spice/analyze.py | 9 ++++----- spice/analyzers/count_lines.py | 3 +++ 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 spice/analyzers/count_lines.py diff --git a/spice/analyze.py b/spice/analyze.py index 8ba9f20..4c323bb 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -9,8 +9,11 @@ from lexers.javascript.javascriptlexer import JavaScriptLexer from lexers.golang.golexer import GoLexer -# gustavo testando alguma coisa +# import analyzer functions from analyzers folder +from spice.analyzers.count_lines import count_lines + +# gustavo testando alguma coisa from spice.analyzers.identation import detect_indentation # this will read the file extension and return the correct lexer @@ -89,10 +92,6 @@ def analyze_file(file_path: str, selected_stats=None): return results -# this will count lines straight from the raw code -def count_lines(code): - return code.count("\n") + 1 - # this will count functions in the AST def count_functions(ast): diff --git a/spice/analyzers/count_lines.py b/spice/analyzers/count_lines.py new file mode 100644 index 0000000..5878bb8 --- /dev/null +++ b/spice/analyzers/count_lines.py @@ -0,0 +1,3 @@ +# this will count lines straight from the raw code +def count_lines(code): + return code.count("\n") + 1 \ No newline at end of file From 37ea44bdd9f9fb273f0aaec388d8a9f556e1f28b Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 11:58:53 -0300 Subject: [PATCH 03/13] make a util module for analyzer related utils --- spice/utils/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 spice/utils/__init__.py diff --git a/spice/utils/__init__.py b/spice/utils/__init__.py new file mode 100644 index 0000000..e69de29 From 9c0319771e67a62da424caa2b2ddb6fe622d3315 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:01:48 -0300 Subject: [PATCH 04/13] make get_lexer its own util file --- spice/analyze.py | 22 +++------------------- spice/utils/get_lexer.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 spice/utils/get_lexer.py diff --git a/spice/analyze.py b/spice/analyze.py index 4c323bb..8419805 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -3,11 +3,9 @@ # this is the universal token, used by all lexers to know what to output from lexers.token import TokenType -# these are the individual lexers for all languages we support -from lexers.ruby.rubylexer import RubyLexer -from lexers.python.pythonlexer import PythonLexer -from lexers.javascript.javascriptlexer import JavaScriptLexer -from lexers.golang.golexer import GoLexer + +# import utils +from spice.utils.get_lexer import get_lexer_for_file # import analyzer functions from analyzers folder from spice.analyzers.count_lines import count_lines @@ -16,20 +14,6 @@ # gustavo testando alguma coisa from spice.analyzers.identation import detect_indentation -# this will read the file extension and return the correct lexer -def get_lexer_for_file(file_path): - _, ext = os.path.splitext(file_path) - - if ext == ".rb": - return RubyLexer - elif ext == ".py": - return PythonLexer - elif ext == ".js": - return JavaScriptLexer - elif ext == ".go": - return GoLexer - else: - raise ValueError(f"Unsupported file extension: {ext}") # this is the analyze function diff --git a/spice/utils/get_lexer.py b/spice/utils/get_lexer.py new file mode 100644 index 0000000..4b595a2 --- /dev/null +++ b/spice/utils/get_lexer.py @@ -0,0 +1,25 @@ +import os + + +# this will read the file extension and return the correct lexer +def get_lexer_for_file(file_path): + _, ext = os.path.splitext(file_path) + + if ext == ".rb": + from lexers.ruby.rubylexer import RubyLexer + return RubyLexer + + elif ext == ".py": + from lexers.python.pythonlexer import PythonLexer + return PythonLexer + + elif ext == ".js": + from lexers.javascript.javascriptlexer import JavaScriptLexer + return JavaScriptLexer + + elif ext == ".go": + from lexers.golang.golexer import GoLexer + return GoLexer + + else: + raise ValueError(f"Unsupported file extension: {ext}") \ No newline at end of file From 16cd5b91256ba13df750ebb027dea60f9b4e7469 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:02:19 -0300 Subject: [PATCH 05/13] remove unused import --- spice/analyze.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spice/analyze.py b/spice/analyze.py index 8419805..affb135 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -1,9 +1,5 @@ import os -# this is the universal token, used by all lexers to know what to output -from lexers.token import TokenType - - # import utils from spice.utils.get_lexer import get_lexer_for_file From 22f2dc49b6701da0ad259a7354de316dd2306def Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:04:37 -0300 Subject: [PATCH 06/13] make count comment lines analyzer its own file --- spice/analyze.py | 20 +------------------- spice/analyzers/count_comment_lines.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 spice/analyzers/count_comment_lines.py diff --git a/spice/analyze.py b/spice/analyze.py index affb135..967d224 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -5,7 +5,7 @@ # import analyzer functions from analyzers folder from spice.analyzers.count_lines import count_lines - +from spice.analyzers.count_comment_lines import count_comment_lines # gustavo testando alguma coisa from spice.analyzers.identation import detect_indentation @@ -120,24 +120,6 @@ def search_node(node): return function_count -# this will count comment lines, since our AST/Parser doesn't include comment lines, this needs to be done in the tokenized output of the lexer -# COMMENT LINE IS A LINE THAT EXCLUSIVELY HAS A COMMENT -# so like: y = 5 #sets y to 5 IS NOT A COMMENT LINE!!!!!!!! -def count_comment_lines(code): - """Count lines that are exclusively comments (no code on the same line)""" - # split the code into lines - lines = code.splitlines() - comment_count = 0 - - for line in lines: - # Remove leading whitespace - stripped = line.strip() - # Check if this line consists only of a comment - if stripped and stripped.startswith('#'): - comment_count += 1 - - return comment_count - def analyze_code_structure(code): indentation_info = detect_indentation(code) diff --git a/spice/analyzers/count_comment_lines.py b/spice/analyzers/count_comment_lines.py new file mode 100644 index 0000000..75914e0 --- /dev/null +++ b/spice/analyzers/count_comment_lines.py @@ -0,0 +1,18 @@ +# this will count comment lines, since our AST/Parser doesn't include comment lines, this needs to be done in the tokenized output of the lexer +# not sure about that first line, im pretty sure like about 200% sure this is analyzing the raw code and not the tokenized code but ok +# COMMENT LINE IS A LINE THAT EXCLUSIVELY HAS A COMMENT +# so like: y = 5 #sets y to 5 IS NOT A COMMENT LINE!!!!!!!! +def count_comment_lines(code): + """Count lines that are exclusively comments (no code on the same line)""" + # split the code into lines + lines = code.splitlines() + comment_count = 0 + + for line in lines: + # Remove leading whitespace + stripped = line.strip() + # Check if this line consists only of a comment + if stripped and stripped.startswith('#'): + comment_count += 1 + + return comment_count \ No newline at end of file From 7cb7546fb1135a9a3cf11d21a5ce93116add7ff4 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:06:28 -0300 Subject: [PATCH 07/13] separate count functions analyzer into own file --- spice/analyze.py | 50 ++---------------------------- spice/analyzers/count_functions.py | 45 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 48 deletions(-) create mode 100644 spice/analyzers/count_functions.py diff --git a/spice/analyze.py b/spice/analyze.py index 967d224..a98596b 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -6,11 +6,12 @@ # import analyzer functions from analyzers folder from spice.analyzers.count_lines import count_lines from spice.analyzers.count_comment_lines import count_comment_lines +from spice.analyzers.count_functions import count_functions # gustavo testando alguma coisa from spice.analyzers.identation import detect_indentation - + # this is the analyze function def analyze_file(file_path: str, selected_stats=None): @@ -73,53 +74,6 @@ def analyze_file(file_path: str, selected_stats=None): -# this will count functions in the AST -def count_functions(ast): - # import function definition from the parser's ast - from parser.ast import FunctionDefinition, Program - - if not isinstance(ast, Program): - return 0 - - function_count = 0 - - # recursive search for function definitions in the AST - def search_node(node): - nonlocal function_count - - if isinstance(node, FunctionDefinition): - function_count += 1 - - # process child nodes if they exist - if hasattr(node, 'statements') and node.statements: - for statement in node.statements: - search_node(statement) - - if hasattr(node, 'body') and node.body: - for body_statement in node.body: - search_node(body_statement) - - # for binary operation, check both sides - if hasattr(node, 'left'): - search_node(node.left) - if hasattr(node, 'right'): - search_node(node.right) - - # check the value part of an assignment - if hasattr(node, 'value'): - search_node(node.value) - - # check function call arguments - if hasattr(node, 'arguments') and node.arguments: - for arg in node.arguments: - search_node(arg) - - # start recursive search from the root Program node - search_node(ast) - - return function_count - - def analyze_code_structure(code): indentation_info = detect_indentation(code) diff --git a/spice/analyzers/count_functions.py b/spice/analyzers/count_functions.py new file mode 100644 index 0000000..5c5d2e8 --- /dev/null +++ b/spice/analyzers/count_functions.py @@ -0,0 +1,45 @@ +# this will count functions in the AST +def count_functions(ast): + # import function definition from the parser's ast + from parser.ast import FunctionDefinition, Program + + if not isinstance(ast, Program): + return 0 + + function_count = 0 + + # recursive search for function definitions in the AST + def search_node(node): + nonlocal function_count + + if isinstance(node, FunctionDefinition): + function_count += 1 + + # process child nodes if they exist + if hasattr(node, 'statements') and node.statements: + for statement in node.statements: + search_node(statement) + + if hasattr(node, 'body') and node.body: + for body_statement in node.body: + search_node(body_statement) + + # for binary operation, check both sides + if hasattr(node, 'left'): + search_node(node.left) + if hasattr(node, 'right'): + search_node(node.right) + + # check the value part of an assignment + if hasattr(node, 'value'): + search_node(node.value) + + # check function call arguments + if hasattr(node, 'arguments') and node.arguments: + for arg in node.arguments: + search_node(arg) + + # start recursive search from the root Program node + search_node(ast) + + return function_count \ No newline at end of file From 415a2f49000d001f20400d3e32408fc8eb91d6ab Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:08:59 -0300 Subject: [PATCH 08/13] add TODO comment for later --- spice/analyze.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spice/analyze.py b/spice/analyze.py index a98596b..0b97c09 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -74,6 +74,13 @@ def analyze_file(file_path: str, selected_stats=None): + + +# im not sure what to do with this part 😂 +# this is the identation analyzer +# but it's not included in the menu? +# im not going to change this since gtins knows better than me how this works +# but this needs to be refactores and included directly into the analyze_file function and the analyze menu def analyze_code_structure(code): indentation_info = detect_indentation(code) @@ -82,4 +89,6 @@ def analyze_code_structure(code): for line, level in indentation_info["levels"]: # print(f"Indentation Level {level}: {line}") print(f"Detected Indentation Type: {indentation_info['indent_type']}") - print(f"Detected Indentation Size: {indentation_info['indent_size']}") \ No newline at end of file + print(f"Detected Indentation Size: {indentation_info['indent_size']}") + +# ---------------------------------------------------------------------------------------------------- \ No newline at end of file From f07ca2936c7e16d68ba3e15b2826389329735c36 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:09:41 -0300 Subject: [PATCH 09/13] remove unused import --- spice/analyzers/identation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spice/analyzers/identation.py b/spice/analyzers/identation.py index d79102b..bb91a27 100644 --- a/spice/analyzers/identation.py +++ b/spice/analyzers/identation.py @@ -1,4 +1,3 @@ -import os import re def detect_indentation(code): From 9e89843238608528a4efe9da6fc592bf66ed34ee Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:12:14 -0300 Subject: [PATCH 10/13] fix comment line count activating parser when its not needed --- spice/analyze.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spice/analyze.py b/spice/analyze.py index 0b97c09..35572fb 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -41,9 +41,14 @@ def analyze_file(file_path: str, selected_stats=None): # line count if requested if "line_count" in selected_stats: results["line_count"] = count_lines(code) + + # comment line count if requested + if "comment_line_count" in selected_stats: + results["comment_line_count"] = count_comment_lines(code) - # only put the code through the lexer and proceed with tokenization if we need function count or comment count (UPDATE THIS WHEN NEEDED PLEASE !!!!!!!!) - if "function_count" in selected_stats or "comment_line_count" in selected_stats: + # only put the code through the lexer and proceed with tokenization if we need function count (UPDATE THIS WHEN NEEDED PLEASE !!!!!!!!) + if "function_count" in selected_stats: + # get the lexer for the code's language LexerClass = get_lexer_for_file(file_path) @@ -51,10 +56,6 @@ def analyze_file(file_path: str, selected_stats=None): lexer = LexerClass(code) tokens = lexer.tokenize() - # process comment line count if requested - if "comment_line_count" in selected_stats: - results["comment_line_count"] = count_comment_lines(code) - # only put the code through the parser and proceed with parsing if we need function count (UPDATE THIS WHEN NEEDED PLEASE !!!!!!!!) if "function_count" in selected_stats: From 5209d0f05f1f91661005a6121add3a7a16ab4e28 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:13:26 -0300 Subject: [PATCH 11/13] fix identation level calling parser when its not needed --- spice/analyze.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spice/analyze.py b/spice/analyze.py index 35572fb..fc31721 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -45,6 +45,10 @@ def analyze_file(file_path: str, selected_stats=None): # comment line count if requested if "comment_line_count" in selected_stats: results["comment_line_count"] = count_comment_lines(code) + + # @gtins botei sua funcao aqui pq ela usa o codigo raw e nao o tokenizado, ai so tirei ela ali de baixo pra nao ficar chamando o parser sem precisar + if "identation_level" in selected_stats: + analyze_code_structure(code) # only put the code through the lexer and proceed with tokenization if we need function count (UPDATE THIS WHEN NEEDED PLEASE !!!!!!!!) if "function_count" in selected_stats: @@ -68,8 +72,6 @@ def analyze_file(file_path: str, selected_stats=None): # count functions results["function_count"] = count_functions(ast) - if "identation_level" in selected_stats: - analyze_code_structure(code) return results From 87176fd41b3242f5ad1ed19a24e4778112c900d4 Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:14:28 -0300 Subject: [PATCH 12/13] add TODO comment @gtins --- spice/analyze.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spice/analyze.py b/spice/analyze.py index fc31721..41dd521 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -47,6 +47,8 @@ def analyze_file(file_path: str, selected_stats=None): results["comment_line_count"] = count_comment_lines(code) # @gtins botei sua funcao aqui pq ela usa o codigo raw e nao o tokenizado, ai so tirei ela ali de baixo pra nao ficar chamando o parser sem precisar + # edit: ok i see whats going on, instead of appending the results to the resuls, this will itself print the results to the terminal + # TODO: make analyze_code_structure return the results, then append those results to the results array if "identation_level" in selected_stats: analyze_code_structure(code) From 46a166b1e864939cb86cc9d707b9adf78e3becaf Mon Sep 17 00:00:00 2001 From: CodyKoInABox Date: Mon, 24 Mar 2025 12:24:27 -0300 Subject: [PATCH 13/13] update imports to only import when necessary --- spice/analyze.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spice/analyze.py b/spice/analyze.py index 41dd521..e8b9ef9 100644 --- a/spice/analyze.py +++ b/spice/analyze.py @@ -1,13 +1,5 @@ import os -# import utils -from spice.utils.get_lexer import get_lexer_for_file - -# import analyzer functions from analyzers folder -from spice.analyzers.count_lines import count_lines -from spice.analyzers.count_comment_lines import count_comment_lines -from spice.analyzers.count_functions import count_functions - # gustavo testando alguma coisa from spice.analyzers.identation import detect_indentation @@ -40,10 +32,12 @@ def analyze_file(file_path: str, selected_stats=None): # line count if requested if "line_count" in selected_stats: + from spice.analyzers.count_lines import count_lines results["line_count"] = count_lines(code) # comment line count if requested if "comment_line_count" in selected_stats: + from spice.analyzers.count_comment_lines import count_comment_lines results["comment_line_count"] = count_comment_lines(code) # @gtins botei sua funcao aqui pq ela usa o codigo raw e nao o tokenizado, ai so tirei ela ali de baixo pra nao ficar chamando o parser sem precisar @@ -56,6 +50,7 @@ def analyze_file(file_path: str, selected_stats=None): if "function_count" in selected_stats: # get the lexer for the code's language + from spice.utils.get_lexer import get_lexer_for_file LexerClass = get_lexer_for_file(file_path) # tokenize the code via lexer @@ -73,6 +68,7 @@ def analyze_file(file_path: str, selected_stats=None): ast = parser.parse() # count functions + from spice.analyzers.count_functions import count_functions results["function_count"] = count_functions(ast) return results