Feature/gdb backtrace with params

assets/example_file.c

@@ -3,3 +3,7 @@
 int add_numbers(int a, int b) {
   return a + b;
 }
+
+int difference_between_numbers(int a, int b) {
+  return a - b;
+}

assets/example_file.h

@@ -3,4 +3,6 @@
 
 int add_numbers(int a, int b);
 
+int difference_between_numbers(int a, int b);
+
 #endif /* EXAMPLE_FILE_H */

docs/CeedlingPacket.md

@@ -985,6 +985,46 @@ that you execute on target hardware).
 
   **Default**: FALSE
 
+  ---
+
+  **Note:**
+
+    The configuration can be combined together with
+
+    ```
+    :test_runner:
+        :cmdline_args: true
+    ```
+
+    After setting **cmdline_args** to **true**, the debuger will execute each test
+    case from crashing test file separately. The exclude and include test_case patterns will
+    be applied, to filter execution of test cases.
+
+    The .gcno and .gcda files will be generated and section of the code under test case
+    causing segmetation fault will be omitted from Coverage Report.
+
+    The default debugger (**gdb**)[https://www.sourceware.org/gdb/] can be switch to any other
+    via setting new configuration under tool node in project.yml. At default set to:
+
+    ```yaml
+      :tools:
+        :backtrace_settings:
+          :executable: gdb
+          :arguments:
+            - -q
+            - --eval-command run
+            - --eval-command backtrace
+            - --batch
+            - --args
+    ```
+
+    Important. The debugger which will collect segmentation fault should run in:
+
+    - background
+    - with option to enable pass to executable binary additional arguments
+
+  ---
+
 * `output`:
 
   The name of your release build binary artifact to be found in <build

lib/ceedling/debugger_utils.rb

@@ -0,0 +1,222 @@
+# The debugger utils class,
+# Store functions and variables helping to parse debugger output and
+# prepare output understandable by report generators
+class DebuggerUtils
+  constructor :configurator,
+              :tool_executor,
+              :unity_utils
+
+  def setup
+    @new_line_tag = '$$$'
+    @colon_tag = '!!!'
+    @command_line = nil
+    @test_result_collector_struct = Struct.new(:passed, :failed, :ignored,
+                                               :output, keyword_init: true)
+  end
+
+  # Copy original command line generated from @tool_executor.build_command_line
+  # to use command line without command line extra args not needed by debugger
+  #
+  # @param [hash, #command] - Command line generated from @tool_executor.build_command_line
+  def configure_debugger(command)
+    # Make a clone of clean command hash
+    # for further calls done for collecting segmentation fault
+    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] &&
+       @configurator.project_config_hash[:test_runner_cmdline_args]
+      @command_line = command.clone
+    elsif @configurator.project_config_hash[:project_use_backtrace_gdb_reporter]
+      # If command_lines are not enabled, do not clone but create reference to command
+      # line
+      @command_line = command
+    end
+  end
+
+  # Execute test_runner file under gdb and return:
+  # - output -> stderr and stdout
+  # - time -> execution of single test
+  #
+  # @param [hash, #command] - Command line generated from @tool_executor.build_command_line
+  # @return [String, #output] - output from binary execution
+  # @return [Float, #time] - time execution of the binary file
+  def collect_cmd_output_with_gdb(command, cmd)
+    gdb_file_name = @configurator.project_config_hash[:tools_backtrace_settings][:executable]
+    gdb_extra_args = @configurator.project_config_hash[:tools_backtrace_settings][:arguments]
+    gdb_extra_args = gdb_extra_args.join(' ')
+
+    gdb_exec_cmd = "#{gdb_file_name} #{gdb_extra_args} #{cmd}"
+    crash_result = @tool_executor.exec(gdb_exec_cmd, command[:options])
+    [crash_result[:output], crash_result[:time].to_f]
+  end
+
+  # Collect list of test cases from test_runner
+  # and apply filters basing at passed :
+  # --test_case
+  # --exclude_test_case
+  # input arguments
+  #
+  # @param [hash, #command] - Command line generated from @tool_executor.build_command_line
+  # @return Array - list of the test_cases defined in test_file_runner
+  def collect_list_of_test_cases(command)
+    all_test_names = command[:line] + @unity_utils.additional_test_run_args('', 'list_test_cases')
+    test_list = @tool_executor.exec(all_test_names, command[:options])
+    test_runner_tc = test_list[:output].split("\n").drop(1)
+
+    # Clean collected test case names
+    # Filter tests which contain test_case_name passed by `--test_case` argument
+    if ENV['CEEDLING_INCLUDE_TEST_CASE_NAME']
+      test_runner_tc.delete_if { |i| !(i =~ /#{ENV['CEEDLING_INCLUDE_TEST_CASE_NAME']}/) }
+    end
+
+    # Filter tests which contain test_case_name passed by `--exclude_test_case` argument
+    if ENV['CEEDLING_EXCLUDE_TEST_CASE_NAME']
+      test_runner_tc.delete_if { |i| i =~ /#{ENV['CEEDLING_EXCLUDE_TEST_CASE_NAME']}/ }
+    end
+
+    test_runner_tc
+  end
+
+  # Update stderr output stream to auto, to collect segmentation fault for
+  # test execution with gcov tool
+  #
+  # @param [hash, #command] - Command line generated from @tool_executor.build_command_line
+  def enable_gcov_with_gdb_and_cmdargs(command)
+    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] &&
+       @configurator.project_config_hash[:test_runner_cmdline_args]
+       command[:options][:stderr_redirect] = if @configurator.project_config_hash[:tools_backtrace_settings][:stderr_redirect] == StdErrRedirect::NONE
+                                               DEFAULT_BACKTRACE_TOOL[:stderr_redirect]
+                                             else
+                                               @configurator.project_config_hash[:tools_backtrace_settings][:stderr_redirect]
+                                             end
+    end
+  end
+
+  # Support function to collect backtrace from gdb.
+  # If test_runner_cmdline_args is set, function it will try to run each of test separately
+  # and create output String similar to non segmentation fault execution but with notification
+  # test with segmentation fault as failure
+  #
+  # @param [hash, #shell_result] - output shell created by calling @tool_executor.exec
+  # @return hash - updated shell_result passed as argument
+  def gdb_output_collector(shell_result)
+    if @configurator.project_config_hash[:test_runner_cmdline_args]
+      test_case_result_collector = @test_result_collector_struct.new(
+        passed: 0,
+        failed: 0,
+        ignored: 0,
+        output: []
+      )
+
+      # Reset time
+      shell_result[:time] = 0
+
+      test_case_list_to_execute = collect_list_of_test_cases(@command_line)
+      test_case_list_to_execute.each do |test_case_name|
+        test_run_cmd = @command_line.clone
+        test_run_cmd_with_args = test_run_cmd[:line] + @unity_utils.additional_test_run_args(test_case_name, 'test_case')
+        test_output, exec_time = collect_cmd_output_with_gdb(test_run_cmd, test_run_cmd_with_args)
+
+        # Concatenate execution time between tests
+        # running tests serpatatelly might increase total execution time
+        shell_result[:time] += exec_time
+
+        # Concatenate test results from single test runs, which not crash
+        # to create proper output for further parser
+        if test_output =~ /([\S]+):(\d+):([\S]+):(IGNORE|PASS|FAIL:)(.*)/
+          test_output = "#{Regexp.last_match(1)}:#{Regexp.last_match(2)}:#{Regexp.last_match(3)}:#{Regexp.last_match(4)}#{Regexp.last_match(5)}"
+          if test_output =~ /:PASS/
+            test_case_result_collector[:passed] += 1
+          elsif test_output =~ /:IGNORE/
+            test_case_result_collector[:ignored] += 1
+          elsif test_output =~ /:FAIL:/
+            test_case_result_collector[:failed] += 1
+          end
+        else
+          # <-- Parse Segmentatation Fault output section -->
+
+          # Withdraw test_name from gdb output
+          test_name = if test_output =~ /<(.*)>/
+                        Regexp.last_match(1)
+                      else
+                        ''
+                      end
+
+          # Collect file_name and line in which Segmentation fault have his beginning
+          if test_output =~ /#{test_name}\s\(\)\sat\s(.*):(\d+)\n/
+            # Remove path from file_name
+            file_name = Regexp.last_match(1).to_s.split('/').last.split('\\').last
+            # Save line number
+            line = Regexp.last_match(2)
+
+            # Replace:
+            # - '\n' by @new_line_tag to make gdb output flat
+            # - ':' by @colon_tag to avoid test results problems
+            # to enable parsing output for default generator_test_results regex
+            test_output = test_output.gsub("\n", @new_line_tag).gsub(':', @colon_tag)
+            test_output = "#{file_name}:#{line}:#{test_name}:FAIL: #{test_output}"
+          end
+
+          # Mark test as failure
+          test_case_result_collector[:failed] += 1
+        end
+        test_case_result_collector[:output].append("#{test_output}\r\n")
+      end
+
+      template = "\n-----------------------\n" \
+                 "\n#{(test_case_result_collector[:passed] + \
+                       test_case_result_collector[:failed] + \
+                       test_case_result_collector[:ignored])} " \
+                  "Tests #{test_case_result_collector[:failed]} " \
+                  "Failures #{test_case_result_collector[:ignored]} Ignored\n\n"
+
+      template += if test_case_result_collector[:failed] > 0
+                    "FAIL\n"
+                  else
+                    "OK\n"
+                  end
+      shell_result[:output] = test_case_result_collector[:output].join('') + \
+                              template
+    else
+      shell_result[:output], shell_result[:time] = collect_cmd_output_with_gdb(
+        @command_line,
+        @command_line[:line]
+      )
+    end
+
+    shell_result
+  end
+
+  # Restore new line under flatten log
+  #
+  # @param(String, #text) - string containing flatten output log
+  # @return [String, #output] - output with restored new line character
+  def restore_new_line_character_in_flatten_log(text)
+    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] &&
+       @configurator.project_config_hash[:test_runner_cmdline_args]
+      text = text.gsub(@new_line_tag, "\n")
+    end
+    text
+  end
+
+  # Restore colon character under flatten log
+  #
+  # @param(String, #text) - string containing flatten output log
+  # @return [String, #output] - output with restored colon character
+  def restore_colon_character_in_flatten_log(text)
+    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] &&
+       @configurator.project_config_hash[:test_runner_cmdline_args]
+      text = text.gsub(@colon_tag, ':')
+    end
+    text
+  end
+
+  # Unflat segmentation fault log
+  #
+  # @param(String, #text) - string containing flatten output log
+  # @return [String, #output] - output with restored colon and new line character
+  def unflat_debugger_log(text)
+    text = restore_new_line_character_in_flatten_log(text)
+    text = restore_colon_character_in_flatten_log(text)
+    text = text.gsub('"',"'") # Replace " character by ' for junit_xml reporter
+    text
+  end
+end

lib/ceedling/defaults.rb

@@ -237,12 +237,28 @@
     ].freeze
   }
 
+DEFAULT_BACKTRACE_TOOL = {
+  :executable => FilePathUtils.os_executable_ext('gdb').freeze,
+  :name => 'default_backtrace_settings'.freeze,
+  :stderr_redirect => StdErrRedirect::AUTO.freeze,
+  :background_exec => BackgroundExec::NONE.freeze,
+  :optional => false.freeze,
+  :arguments => [
+    '-q',
+    '--eval-command run',
+    '--eval-command backtrace',
+    '--batch',
+    '--args'
+    ].freeze
+  }
+
 
 DEFAULT_TOOLS_TEST = {
   :tools => {
     :test_compiler => DEFAULT_TEST_COMPILER_TOOL,
     :test_linker   => DEFAULT_TEST_LINKER_TOOL,
     :test_fixture  => DEFAULT_TEST_FIXTURE_TOOL,
+    :backtrace_settings => DEFAULT_BACKTRACE_TOOL,
     }
   }
 
@@ -386,7 +402,8 @@
     },
 
     # all tools populated while building up config structure
-    :tools => {},
+    :tools => {
+    },
 
     # empty argument lists for default tools
     # (these can be overridden in project file to add arguments to tools without totally redefining tools)

lib/ceedling/generator.rb

@@ -17,6 +17,7 @@ class Generator
               :streaminator,
               :plugin_manager,
               :file_wrapper,
+              :debugger_utils,
               :unity_utils
 
 
@@ -166,28 +167,36 @@ def generate_test_results(tool, context, executable, result)
     @plugin_manager.pre_test_fixture_execute(arg_hash)
 
     @streaminator.stdout_puts("Running #{File.basename(arg_hash[:executable])}...", Verbosity::NORMAL)
-
     # Unity's exit code is equivalent to the number of failed tests, so we tell @tool_executor not to fail out if there are failures
     # so that we can run all tests and collect all results
     command = @tool_executor.build_command_line(arg_hash[:tool], [], arg_hash[:executable])
+
+    # Configure debugger
+    @debugger_utils.configure_debugger(command)
+
+    # Apply additional test case filters 
     command[:line] += @unity_utils.collect_test_runner_additional_args
     @streaminator.stdout_puts("Command: #{command}", Verbosity::DEBUG)
+
+    # Enable collecting GCOV results even when segmenatation fault is appearing
+    # The gcda and gcno files will be generated for a test cases which doesn't
+    # cause segmentation fault
+    @debugger_utils.enable_gcov_with_gdb_and_cmdargs(command)
+
     command[:options][:boom] = false
     shell_result = @tool_executor.exec( command[:line], command[:options] )
 
     shell_result[:exit_code] = 0
+
     # Add extra collecting backtrace
     # if use_backtrace_gdb_reporter is set to true
-    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] and (shell_result[:output] =~ /\s*Segmentation\sfault.*/)
-      gdb_file_name = FilePathUtils.os_executable_ext('gdb').freeze
-      gdb_exec_cmd = "#{gdb_file_name} -q #{command[:line]} --eval-command run --eval-command backtrace --batch"
-      crash_result = @tool_executor.exec(gdb_exec_cmd, command[:options] )
-      shell_result[:output] = crash_result[:output]
+    if @configurator.project_config_hash[:project_use_backtrace_gdb_reporter] &&
+       (shell_result[:output] =~ /\s*Segmentation\sfault.*/)
+      shell_result = @debugger_utils.gdb_output_collector(shell_result)
     else
       # Don't Let The Failure Count Make Us Believe Things Aren't Working
       @generator_helper.test_results_error_handler(executable, shell_result)
     end
-
     processed = @generator_test_results.process_and_write_results( shell_result,
                                                                    arg_hash[:result_file],
                                                                    @file_finder.find_test_from_file_path(arg_hash[:executable]) )