Various fixes for 4.5.0

.github/workflows/ci.yml

@@ -272,11 +272,8 @@ jobs:
           python-version: '3.13'
 
       - run: |
-          KO=0
-          python3 tools/ark_frequent_instructions.py super_insts_usage > output.txt || KO=1
-          echo "SUPER_INSTS_REPORT_KO=$KO" >> $GITHUB_ENV
           echo "SUPER_INSTS_REPORT<<EOF" >> $GITHUB_ENV
-          cat output.txt >> $GITHUB_ENV
+          python3 tools/ark_frequent_instructions.py super_insts_usage >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
 
       - uses: 8BitJonny/gh-get-current-pr@4.0.0

CHANGELOG.md

@@ -6,8 +6,10 @@
 ### Deprecations
 
 ### Added
+- added new macro `$gensym`, to generate a unique symbol identifier to use in macros
 
 ### Changed
+- all paths inside `if` should return a value, when used as an expression. If an `else` branch is missing, `nil` will be returned
 
 ### Removed
 

docs/arkdoc/Builtins.txt

@@ -115,3 +115,15 @@
 * (print (hasField closure "B"))  # false, field names are case-sensitive
 * =end
 #--
+
+--#
+* @name apply
+* @brief Call a function with a list of arguments
+* @param f function
+* @param args list, can be empty if the function takes no argument
+* =begin
+* (print (apply + [1 2]))  # 3
+* (let foo (fun (a b c) (+ a b c)))
+* (print (apply foo [1 2 3]))  # 6
+* =end
+#--

docs/arkdoc/Macros.txt

@@ -122,3 +122,14 @@
 * (print (one 1 5 6 7 8))  # 5
 * =end
 #--
+
+--#
+* @name $gensym
+* @brief Return a new unique symbol identifier to use in macros
+* =begin
+* (macro switch (value case then ...cases) {
+*   (macro var ($gensym))
+*   (let var value)
+*   (_switch_impl var case then ...cases) })
+* =end
+#--

include/Ark/Compiler/Lowerer/ASTLowerer.hpp

@@ -98,6 +98,12 @@ namespace Ark::internal
             bool is_temp;
         };
 
+        struct Var
+        {
+            std::string name;
+            std::size_t argument_count;
+        };
+
         LocalsLocator m_locals_locator;
 
         // tables: symbols, values, plugins and codes
@@ -107,7 +113,7 @@ namespace Ark::internal
         std::vector<IR::Block> m_code_pages;
         std::vector<IR::Block> m_temp_pages;  ///< we need temporary code pages for some compilations passes
         IR::label_t m_current_label = 0;
-        std::stack<std::string> m_opened_vars;  ///< stack of vars we are currently declaring
+        std::stack<Var> m_opened_vars;  ///< stack of vars we are currently declaring
 
         enum class ErrorKind
         {
@@ -150,7 +156,7 @@ namespace Ark::internal
          */
         [[nodiscard]] bool isFunctionCallingItself(const std::string& name) noexcept
         {
-            return !m_opened_vars.empty() && m_opened_vars.top() == name;
+            return !m_opened_vars.empty() && m_opened_vars.top().name == name;
         }
 
         /**

include/Ark/Compiler/Macros/Processor.hpp

@@ -61,6 +61,7 @@ namespace Ark::internal
         std::shared_ptr<MacroExecutor> m_conditional_executor;
         std::vector<std::shared_ptr<MacroExecutor>> m_executors;
         std::unordered_map<std::string, Node> m_defined_functions;
+        std::size_t m_genned_sym;
 
         /**
          * @brief Return std::nullopt if the function isn't registered, otherwise return its node

src/arkreactor/Compiler/Lowerer/ASTLowerer.cpp

@@ -5,6 +5,7 @@
 #include <utility>
 #include <algorithm>
 #include <fmt/core.h>
+#include <fmt/ranges.h>
 
 #include <Ark/Error/Exceptions.hpp>
 #include <Ark/Error/Diagnostics.hpp>
@@ -473,6 +474,11 @@ namespace Ark::internal
             page(p).back().setSourceLocation(x.constList()[3].filename(), x.constList()[3].position().start.line);
             m_locals_locator.dropVarsForBranch();
         }
+        else
+        {
+            Node tmp = Node(NodeType::List);
+            compileExpression(tmp, p, is_result_unused, is_terminal);
+        }
 
         // when else is finished, jump to end
         const auto label_end = IR::Entity::Label(m_current_label++);
@@ -531,18 +537,21 @@ namespace Ark::internal
         // save page_id into the constants table as PageAddr and load the const
         page(p).emplace_back(is_closure ? MAKE_CLOSURE : LOAD_CONST, addValue(function_body_page.index, x));
 
+        std::size_t arg_count = 0;
         // pushing arguments from the stack into variables in the new scope
         for (const auto& node : x.constList()[1].constList() | std::ranges::views::reverse)
         {
             if (node.nodeType() == NodeType::Symbol || node.nodeType() == NodeType::MutArg)
             {
                 page(function_body_page).emplace_back(STORE, addSymbol(node));
                 m_locals_locator.addLocal(node.string());
+                arg_count++;
             }
             else if (node.nodeType() == NodeType::RefArg)
             {
                 page(function_body_page).emplace_back(STORE_REF, addSymbol(node));
                 m_locals_locator.addLocal(node.string());
+                arg_count++;
             }
         }
 
@@ -551,7 +560,7 @@ namespace Ark::internal
         // (let name (fun (e) (map lst (fun (e) (name e)))))
         // Otherwise, `name` would have been optimized to a CALL_CURRENT_PAGE, which would have returned the wrong page.
         if (x.isAnonymousFunction())
-            m_opened_vars.emplace("#anonymous");
+            m_opened_vars.emplace("#anonymous", arg_count);
         // push body of the function
         compileExpression(x.list()[2], function_body_page, false, true);
         if (x.isAnonymousFunction())
@@ -579,13 +588,23 @@ namespace Ark::internal
         const std::string name = x.constList()[1].string();
         uint16_t i = addSymbol(x.constList()[1]);
 
-        if (!m_opened_vars.empty() && m_opened_vars.top() == name)
+        if (!m_opened_vars.empty() && m_opened_vars.top().name == name)
             buildAndThrowError("Can not define a variable using the same name as the function it is defined inside. You need to rename the function or the variable", x);
 
         const bool is_function = x.constList()[2].isFunction();
         if (is_function)
         {
-            m_opened_vars.push(name);
+            std::size_t arg_count = 0;
+            if (x.constList()[2].nodeType() == NodeType::List && x.constList()[2].constList().size() >= 2 &&
+                x.constList()[2].constList()[1].nodeType() == NodeType::List)
+            {
+                for (const auto& node : x.constList()[2].constList()[1].constList())
+                {
+                    if (node.nodeType() == NodeType::Symbol || node.nodeType() == NodeType::MutArg || node.nodeType() == NodeType::RefArg)
+                        arg_count++;
+                }
+            }
+            m_opened_vars.push(Var(name, arg_count));
             x.list()[2].setFunctionKind(/* anonymous= */ false);
         }
 
@@ -815,10 +834,41 @@ namespace Ark::internal
         constexpr std::size_t start_index = 1;
         Node& node = x.list()[0];
 
+        // number of arguments
+        std::size_t args_count = 0;
+        for (auto it = x.constList().begin() + start_index, it_end = x.constList().end(); it != it_end; ++it)
+        {
+            if (it->nodeType() != NodeType::Capture && !isBreakpoint(*it))
+                args_count++;
+        }
+
         if (is_terminal && node.nodeType() == NodeType::Symbol && isFunctionCallingItself(node.string()))
         {
             pushFunctionCallArguments(x, p, /* is_tail_call= */ true);
 
+            if (const std::size_t expected_arg_count = m_opened_vars.top().argument_count; args_count != expected_arg_count)
+            {
+                std::vector<std::string> arg_names;
+                if (expected_arg_count > 0)
+                {
+                    arg_names.reserve(expected_arg_count + 1);
+                    arg_names.emplace_back("");
+                    for (std::size_t i = 0; i < expected_arg_count; ++i)
+                        arg_names.emplace_back(1, static_cast<char>('a' + i));
+                }
+
+                buildAndThrowError(
+                    fmt::format(
+                        "When performing tail-call `{}', received {} argument{}, but expected {}: `({}{})'",
+                        x.repr(),
+                        args_count,
+                        args_count > 1 ? "s" : "",
+                        expected_arg_count,
+                        node.string(),
+                        fmt::join(arg_names, " ")),
+                    x);
+            }
+
             // jump to the top of the function
             page(p).emplace_back(TAIL_CALL_SELF);
             page(p).back().setSourceLocation(node.filename(), node.position().start.line);
@@ -889,14 +939,6 @@ namespace Ark::internal
 
         pushFunctionCallArguments(x, p, /* is_tail_call= */ false);
 
-        // number of arguments
-        std::size_t args_count = 0;
-        for (auto it = x.constList().begin() + start_index, it_end = x.constList().end(); it != it_end; ++it)
-        {
-            if (it->nodeType() != NodeType::Capture && !isBreakpoint(*it))
-                args_count++;
-        }
-
         // call the procedure
         switch (call_type)
         {

src/arkreactor/Compiler/Macros/Processor.cpp

@@ -18,7 +18,7 @@
 namespace Ark::internal
 {
     MacroProcessor::MacroProcessor(const unsigned debug) noexcept :
-        Pass("MacroProcessor", debug)
+        Pass("MacroProcessor", debug), m_genned_sym(0)
     {
         // create executors pipeline
         m_conditional_executor = std::make_shared<ConditionalExecutor>(this);
@@ -518,6 +518,12 @@ namespace Ark::internal
                     node.push_back(getListNode());
                 }
             }
+            else if (name == "$gensym")
+            {
+                checkMacroArgCountEq(node, 0, "$gensym", true);
+                node.updateValueAndType(Node(NodeType::Symbol, fmt::format("#gensym-{}", m_genned_sym)));
+                ++m_genned_sym;
+            }
             else if (name == Language::Symcat)
             {
                 if (node.list().size() <= 2)

tests/unittests/resources/CompilerSuite/optimized_ir/type.expected

@@ -40,6 +40,7 @@ page_1
 .L7:
 	CHECK_TYPE_OF 0, 1
 	POP_JUMP_IF_TRUE L9
+	BUILTIN 2
 	JUMP L10
 .L9:
 	PUSH_RETURN_ADDRESS L11

tests/unittests/resources/DiagnosticsSuite/compileTime/macro_gensym_too_many_args.ark

@@ -0,0 +1 @@
+(macro a ($gensym 1))

tests/unittests/resources/DiagnosticsSuite/compileTime/macro_gensym_too_many_args.expected

@@ -0,0 +1,5 @@
+In file tests/unittests/resources/DiagnosticsSuite/compileTime/macro_gensym_too_many_args.ark:1
+    1 | (macro a ($gensym 1))
+      |          ^~~~~~~~~~~
+    2 |
+        When expanding `$gensym' inside a macro, got 1 argument, expected 0

tests/unittests/resources/DiagnosticsSuite/compileTime/too_many_args_tail_call.ark

@@ -0,0 +1,3 @@
+(let f (fun(i)
+  (f (print i) (+ 1 i))))
+(f 1)

tests/unittests/resources/DiagnosticsSuite/compileTime/too_many_args_tail_call.expected

@@ -0,0 +1,7 @@
+In file tests/unittests/resources/DiagnosticsSuite/compileTime/too_many_args_tail_call.ark:2
+    1 | (let f (fun(i)
+    2 |   (f (print i) (+ 1 i))))
+      |   ^~~~~~~~~~~~~~~~~~~~~
+    3 | (f 1)
+    4 |
+        When performing tail-call `(f (print i) (+ 1 i))', received 2 arguments, but expected 1: `(f a)'

tests/unittests/resources/LangSuite/weird-tests.ark

@@ -54,4 +54,13 @@
   (test:case "(f4 (mut n 1) (+ 0 n) n (set n 2))" {
     (mut output [])
     (f4 (mut n 1) (+ 0 n) n (set n 2))
-    (test:eq output [1 1 2 2]) }) })
+    (test:eq output [1 1 2 2]) })
+
+  # not all paths in `or` return, and we have a POP at the end, that would pop the function and leave
+  # the instruction pointer alone, messing up the call stack
+  (test:case "(or false (if 0 0))" {
+    (let g (fun (a b) {
+      (or false (if 0 0))
+      b }))
+
+    (test:eq (g 1 2) 2) }) })

tools/ark_frequent_instructions.py

@@ -149,26 +149,31 @@ def print_most_freqs(data, max_percent=10):
     most = sorted(data.items(), key=lambda e: e[1], reverse=True)
     interesting = most[:(len(most) * max_percent) // 100]
     if compute_super_insts_usage:
+        threshold = 10
+        over, under = [(x, c) for (x, c) in most if c > threshold], [(x, c) for (x, c) in most if c <= threshold]
+
+        if under:
+            print(f"Some Super Instructions are under the usage threshold ({threshold}).\n")
+            print("| Super Instruction | Uses in compiled code |")
+            print("| ----------------- | --------------------- |")
+            print("\n".join(f"| {insts} | {count} |" for (insts, count) in under))
+
+        print("<details><summary>Super Instructions over the threshold</summary>\n")
         print("| Super Instruction | Uses in compiled code |")
         print("| ----------------- | --------------------- |")
-        print("\n".join(f"| {insts} | {count} |" for (insts, count) in interesting))
+        print("\n".join(f"| {insts} | {count} |" for (insts, count) in over))
+        print("\n</details>")
     else:
         print("\n".join(f"{insts} -> {count}" for (insts, count) in interesting))
 
-    if compute_super_insts_usage:
-        threshold = 10
-        for (inst, count) in most:
-            if count <= threshold:
-                sys.exit(1)
-
-
-if not compute_super_insts_usage:
-    print("Super instructions present:")
-print_most_freqs(super_insts_freqs, max_percent=100)
 
 if compute_super_insts_usage:
-    sys.exit(0)
+    print_most_freqs(super_insts_freqs, max_percent=100)
+else:
+    print("Super instructions present:")
+    print_most_freqs(super_insts_freqs, max_percent=100)
 
-for i in (2, 3, 4):
-    print(f"\nPairs of {i}:")
-    print_most_freqs(frequent[i])
+    print("Potential pairs of instructions that could be optimized:")
+    for i in (2, 3, 4):
+        print(f"\nPairs of {i}:")
+        print_most_freqs(frequent[i])