[UT] Add unit tests to cover extended debugger

Add UT to cover:

- collect test_case results from crashing file and other files
- collect test_case result filtered by --test_case from crashing file
- collect test_case result filtered by --exclude_test_case from crashing file

Author: luze-mobica

assets/example_file.c

@@ -3,3 +3,7 @@
 int add_numbers(int a, int b) {
   return a + b;
 }
+
+int difference_between_numbers(int a, int b) {
+  return a - b;
+}

assets/example_file.h

@@ -3,4 +3,6 @@
 
 int add_numbers(int a, int b);
 
+int difference_between_numbers(int a, int b);
+
 #endif /* EXAMPLE_FILE_H */

spec/gcov/gcov_deployment_spec.rb

@@ -32,6 +32,9 @@
       it { can_test_projects_with_gcov_with_compile_error }
       it { can_fetch_project_help_for_gcov }
       it { can_create_html_report }
+      it { can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and_cmd_args_set_to_true_for_test_cases_not_causing_crash }
+      it { can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and_cmd_args_set_to_true_with_zero_coverage }
+      it { can_create_gcov_html_report_from_test_runner_with_enabled_debug_and_cmd_args_set_to_true_with_100_coverage_when_excluding_crashing_test_case }
     end
 
 

spec/gcov/gcov_test_cases_spec.rb

@@ -194,4 +194,147 @@ def can_create_html_report
       end
     end
   end
+
+  def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and_cmd_args_set_to_true_for_test_cases_not_causing_crash
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+        FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml'
+
+        add_line = false
+        updated_prj_yml = []
+        File.read('project.yml').split("\n").each do |line|
+          if line =~ /\:project\:/
+            add_line = true
+            updated_prj_yml.append(line)
+          else
+            if add_line
+              updated_prj_yml.append('  :use_backtrace_gdb_reporter: TRUE')
+              add_line = false
+            end
+            updated_prj_yml.append(line)
+          end
+        end
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling gcov:all 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
+        gcov_html_report = `bundle exec ruby -S ceedling utils:gcov 2>&1`
+        expect($?.exitstatus).to match(0)
+        expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.fail'))
+        output_rd = File.read('./build/gcov/results/test_example_file_sigsegv.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+1/)
+        expect(output).to match(/IGNORED:\s+0/)
+        expect(output).to match(/example_file.c Lines executed:50.00% of 4/)
+
+        expect(gcov_html_report).to match(/Creating gcov results report\(s\) in 'build\/artifacts\/gcov'\.\.\. Done/)
+        expect(File.exist?('build/artifacts/gcov/GcovCoverageResults.html')).to eq true
+      end
+    end
+  end
+
+  def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and_cmd_args_set_to_true_with_zero_coverage
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+        FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml'
+
+        add_line = false
+        updated_prj_yml = []
+        File.read('project.yml').split("\n").each do |line|
+          if line =~ /\:project\:/
+            add_line = true
+            updated_prj_yml.append(line)
+          else
+            if add_line
+              updated_prj_yml.append('  :use_backtrace_gdb_reporter: TRUE')
+              add_line = false
+            end
+            updated_prj_yml.append(line)
+          end
+        end
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling gcov:all --exclude_test_case=test_add_numbers_adds_numbers 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
+        gcov_html_report = `bundle exec ruby -S ceedling utils:gcov 2>&1`
+        expect($?.exitstatus).to match(0)
+        expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.fail'))
+        output_rd = File.read('./build/gcov/results/test_example_file_sigsegv.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+0/)
+        expect(output).to match(/FAILED:\s+1/)
+        expect(output).to match(/IGNORED:\s+0/)
+        expect(output).to match(/example_file.c Lines executed:0.00% of 4/)
+
+        expect(gcov_html_report).to match(/Creating gcov results report\(s\) in 'build\/artifacts\/gcov'\.\.\. Done/)
+        expect(File.exist?('build/artifacts/gcov/GcovCoverageResults.html')).to eq true
+      end
+    end
+  end
+
+  def can_create_gcov_html_report_from_test_runner_with_enabled_debug_and_cmd_args_set_to_true_with_100_coverage_when_excluding_crashing_test_case
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+        FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml'
+
+        add_line = false
+        updated_prj_yml = File.read('project.yml').split("\n")
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        add_test_case = "\nvoid test_difference_between_two_numbers(void)\n"\
+                        "{\n" \
+                        "  TEST_ASSERT_EQUAL(0, difference_between_numbers(1,1));\n" \
+                        "}\n"
+        
+        updated_test_file = File.read('test/test_example_file_sigsegv.c').split("\n")
+        updated_test_file.insert(updated_test_file.length(), add_test_case)
+        File.write('test/test_example_file_sigsegv.c', updated_test_file.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling gcov:all --exclude_test_case=test_add_numbers_will_fail 2>&1`
+        expect($?.exitstatus).to match(0)
+        gcov_html_report = `bundle exec ruby -S ceedling utils:gcov 2>&1`
+        expect($?.exitstatus).to match(0)
+        expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.pass'))
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+2/)
+        expect(output).to match(/FAILED:\s+0/)
+        expect(output).to match(/IGNORED:\s+0/)
+        expect(output).to match(/example_file.c Lines executed:100.00% of 4/)
+
+        expect(gcov_html_report).to match(/Creating gcov results report\(s\) in 'build\/artifacts\/gcov'\.\.\. Done/)
+        expect(File.exist?('build/artifacts/gcov/GcovCoverageResults.html')).to eq true
+      end
+    end
+  end
 end

spec/generator_test_results_spec.rb

@@ -5,6 +5,7 @@
 require 'ceedling/constants'
 require 'ceedling/streaminator'
 require 'ceedling/configurator'
+require 'ceedling/debugger_utils'
 
 NORMAL_OUTPUT =
   "Verbose output one\n" +
@@ -61,8 +62,16 @@
     # these will always be used as is.
     @yaml_wrapper = YamlWrapper.new
     @sanity_checker = GeneratorTestResultsSanityChecker.new({:configurator => @configurator, :streaminator => @streaminator})
-    
-    @generate_test_results = described_class.new({:configurator => @configurator, :generator_test_results_sanity_checker => @sanity_checker, :yaml_wrapper => @yaml_wrapper})
+    @debugger_utils = DebuggerUtils.new({:configurator => @configurator, :tool_executor => nil, :unity_utils => nil})
+
+    @generate_test_results = described_class.new(
+      {
+        :configurator => @configurator,
+        :generator_test_results_sanity_checker => @sanity_checker,
+        :yaml_wrapper => @yaml_wrapper,
+        :debugger_utils => @debugger_utils
+      }
+    )
   end
 
   after(:each) do

spec/spec_system_helper.rb

@@ -843,7 +843,7 @@ def test_run_of_projects_fail_because_of_sigsegv_without_report
         expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
         expect(output).to match(/Segmentation fault \(core dumped\)/)
         expect(output).to match(/No tests executed./)
-        expect(!File.exists?('./build/test/results/test_add.fail'))
+        expect(!File.exist?('./build/test/results/test_add.fail'))
       end
     end
   end
@@ -876,13 +876,142 @@ def test_run_of_projects_fail_because_of_sigsegv_with_report
         expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
         expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
         expect(output).to match(/Unit test failures./)
-        expect(File.exists?('./build/test/results/test_example_file_sigsegv.fail'))
+        expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail'))
         output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail')
         expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
       end
     end
   end
 
+  def execute_all_test_cases_from_crashing_test_runner_and_return_test_report_with_failue_when_cmd_args_set_to_true
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+
+        add_line = false
+        updated_prj_yml = []
+        File.read('project.yml').split("\n").each do |line|
+          if line =~ /\:project\:/
+            add_line = true
+            updated_prj_yml.append(line)
+          else
+            if add_line
+              updated_prj_yml.append('  :use_backtrace_gdb_reporter: TRUE')
+              add_line = false
+            end
+            updated_prj_yml.append(line)
+          end
+        end
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling test:all 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
+        expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
+        expect(output).to match(/TESTED:\s+2/)
+        expect(output).to match(/PASSED:\s+1/)
+        expect(output).to match(/FAILED:\s+1/)
+        expect(output).to match(/IGNORED:\s+0/)
+      end
+    end
+  end
+
+  def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_test_case_argument_with_enabled_debug_and_cmd_args_set_to_true
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+
+        add_line = false
+        updated_prj_yml = []
+        File.read('project.yml').split("\n").each do |line|
+          if line =~ /\:project\:/
+            add_line = true
+            updated_prj_yml.append(line)
+          else
+            if add_line
+              updated_prj_yml.append('  :use_backtrace_gdb_reporter: TRUE')
+              add_line = false
+            end
+            updated_prj_yml.append(line)
+          end
+        end
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling test:all --test_case=test_add_numbers_will_fail 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
+        expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+0/)
+        expect(output).to match(/FAILED:\s+1/)
+        expect(output).to match(/IGNORED:\s+0/)
+      end
+    end
+  end
+
+  def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_exclude_test_case_argument_with_enabled_debug_and_cmd_args_set_to_true
+    @c.with_context do
+      Dir.chdir @proj_name do
+        FileUtils.cp test_asset_path("example_file.h"), 'src/'
+        FileUtils.cp test_asset_path("example_file.c"), 'src/'
+        FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/'
+
+        add_line = false
+        updated_prj_yml = []
+        File.read('project.yml').split("\n").each do |line|
+          if line =~ /\:project\:/
+            add_line = true
+            updated_prj_yml.append(line)
+          else
+            if add_line
+              updated_prj_yml.append('  :use_backtrace_gdb_reporter: TRUE')
+              add_line = false
+            end
+            updated_prj_yml.append(line)
+          end
+        end
+        enable_unity_extra_args = "\n:test_runner:\n"\
+                                  "  :cmdline_args: true\n"
+
+        updated_prj_yml.insert(updated_prj_yml.length() -1, enable_unity_extra_args)
+
+        File.write('project.yml', updated_prj_yml.join("\n"), mode: 'w')
+
+        output = `bundle exec ruby -S ceedling test:all --exclude_test_case=add_numbers_adds_numbers 2>&1`
+        expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called
+        expect(output).to match(/Program received signal SIGSEGV, Segmentation fault./)
+        expect(output).to match(/Unit test failures./)
+        expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail'))
+        output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail')
+        expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ )
+        expect(output).to match(/TESTED:\s+1/)
+        expect(output).to match(/PASSED:\s+0/)
+        expect(output).to match(/FAILED:\s+1/)
+        expect(output).to match(/IGNORED:\s+0/)
+      end
+    end
+  end
+
   def can_test_projects_with_success_when_space_appears_between_hash_and_include
     # test case cover issue described in https://github.com/ThrowTheSwitch/Ceedling/issues/588
     @c.with_context do

spec/system/deployment_spec.rb

@@ -49,6 +49,9 @@
     it { handles_destroying_a_module_that_does_not_exist_using_the_module_plugin_path_extension }
     it { test_run_of_projects_fail_because_of_sigsegv_without_report }
     it { test_run_of_projects_fail_because_of_sigsegv_with_report }
+    it { execute_all_test_cases_from_crashing_test_runner_and_return_test_report_with_failue_when_cmd_args_set_to_true }
+    it { execute_and_collect_debug_logs_from_crashing_test_case_defined_by_test_case_argument_with_enabled_debug_and_cmd_args_set_to_true }
+    it { execute_and_collect_debug_logs_from_crashing_test_case_defined_by_exclude_test_case_argument_with_enabled_debug_and_cmd_args_set_to_true }
     it { can_run_single_test_with_full_test_case_name_from_test_file_with_success_cmdline_args_are_enabled }
     it { can_run_single_test_with_partiall_test_case_name_from_test_file_with_enabled_cmdline_args_success }
     it { exlcude_test_case_name_filter_works_and_only_one_test_case_is_executed }