From 68d9bc36b970c999ec6223993ff055a5baed1d81 Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Fri, 28 Apr 2023 13:49:58 +0100 Subject: [PATCH 1/6] Create initial JSON schema for serialised test information --- ruby/Gemfile | 1 + ruby/serialisation/test_schema.json | 153 ++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 ruby/serialisation/test_schema.json diff --git a/ruby/Gemfile b/ruby/Gemfile index cee4cab..527c035 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -2,6 +2,7 @@ source 'https://rubygems.org' group :development, :test do gem "bundler" + gem "json-schema", require: false gem "rake" gem "minitest" gem "rspec" diff --git a/ruby/serialisation/test_schema.json b/ruby/serialisation/test_schema.json new file mode 100644 index 0000000..0198b8c --- /dev/null +++ b/ruby/serialisation/test_schema.json @@ -0,0 +1,153 @@ +{ + "definitions": { + "testItem": { + "type": "object", + "required": [ + "id", + "label", + "uri", + "range" + ], + "properties": { + "id": { + "$ref": "#/definitions/testId" + }, + "description": { + "$ref": "#/definitions/testDescription" + }, + "label": { + "$ref": "#/definitions/testLabel" + }, + "sortText": { + "$ref": "#/definitions/testSortText" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/testTag" + } + }, + "uri": { + "$ref": "#/definitions/testUri" + }, + "error": { + "$ref": "#/definitions/testError" + }, + "range": { + "$ref": "#/definitions/range" + } + }, + "additionalProperties": false + }, + "testGroup": { + "type": "object", + "required": [ + "id", + "label", + "children", + "range" + ], + "properties": { + "id": { + "$ref": "#/definitions/testId" + }, + "description": { + "$ref": "#/definitions/testDescription" + }, + "label": { + "$ref": "#/definitions/testLabel" + }, + "sortText": { + "$ref": "#/definitions/testSortText" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/testTag" + } + }, + "uri": { + "$ref": "#/definitions/testUri" + }, + "error": { + "$ref": "#/definitions/testError" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/definitions/testItem" + } + }, + "range": { + "$ref": "#/definitions/range" + } + }, + "additionalProperties": false + }, + "position": { + "type": "object", + "required": [ + "line", + "character" + ], + "properties": { + "line": { + "type": "integer" + }, + "character": { + "type": "integer" + } + } + }, + "range": { + "type": "object", + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "$ref": "#/definitions/position" + }, + "end": { + "$ref": "#/definitions/position" + } + } + }, + "testId": { + "type": "string", + "description": "Identifier for the test/group. Must be unique among its parent's direct children" + }, + "testLabel": { + "type": "string", + "description": "Display name describing the test case/group" + }, + "testDescription": { + "type": "string", + "description": "Optional description that appears next to the label" + }, + "testSortText": { + "type": "string", + "description": "A string that should be used when comparing this item with other items. When falsy the label is used" + }, + "testTag": { + "type": "object", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "description": "ID of the test tag. TestTag instances with the same ID are considered to be identical" + } + }, + "additionalProperties": false + }, + "testUri": { + "type": "string", + "description": "URI this TestItem is associated with. May be a file or directory" + }, + "testError": { + "type": "string", + "description": "Optional error encountered while loading the test. Note that this is not a test result and should only be used to represent errors in test discovery, such as syntax errors" + } + } +} From 64429141d0bdd517ccb4e2c4e4ed0fe423aebaf3 Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Mon, 8 May 2023 17:50:03 +0100 Subject: [PATCH 2/6] Tweak status and add boilerplate classes for generating JSON test data --- ruby/serialisation/location.rb | 29 ++++++++ ruby/serialisation/position.rb | 28 ++++++++ ruby/serialisation/range.rb | 28 ++++++++ ruby/serialisation/status/base.rb | 44 ++++++++++++ ruby/serialisation/status/enqueued.rb | 19 +++++ ruby/serialisation/status/errored.rb | 19 +++++ ruby/serialisation/status/failed.rb | 19 +++++ ruby/serialisation/status/passed.rb | 19 +++++ ruby/serialisation/status/skipped.rb | 19 +++++ ruby/serialisation/status/started.rb | 19 +++++ ruby/serialisation/status/test_message.rb | 42 +++++++++++ ruby/serialisation/test_item.rb | 63 ++++++++++++++++ ruby/serialisation/test_schema.json | 87 ++++++++++++++++------- 13 files changed, 408 insertions(+), 27 deletions(-) create mode 100644 ruby/serialisation/location.rb create mode 100644 ruby/serialisation/position.rb create mode 100644 ruby/serialisation/range.rb create mode 100644 ruby/serialisation/status/base.rb create mode 100644 ruby/serialisation/status/enqueued.rb create mode 100644 ruby/serialisation/status/errored.rb create mode 100644 ruby/serialisation/status/failed.rb create mode 100644 ruby/serialisation/status/passed.rb create mode 100644 ruby/serialisation/status/skipped.rb create mode 100644 ruby/serialisation/status/started.rb create mode 100644 ruby/serialisation/status/test_message.rb create mode 100644 ruby/serialisation/test_item.rb diff --git a/ruby/serialisation/location.rb b/ruby/serialisation/location.rb new file mode 100644 index 0000000..c10e909 --- /dev/null +++ b/ruby/serialisation/location.rb @@ -0,0 +1,29 @@ +# # frozen_string_literal: true + +require "json" +require "uri" + +module Serialisation + class Location + def initialize(uri:, range:) + raise ArgumentError, "uri must be a URI" unless uri.is_a?(URI::Generic) + raise ArgumentError, "range must be a Range" unless range.is_a?(Serialisation::Range) + + @uri = uri + @range = range + end + + attr_reader :uri, :range + + def as_json(*) + { + "uri" => uri.to_s, + "range" => range.as_json, + } + end + + def to_json(*args) + as_json.to_json(*args) + end + end +end diff --git a/ruby/serialisation/position.rb b/ruby/serialisation/position.rb new file mode 100644 index 0000000..ecebd22 --- /dev/null +++ b/ruby/serialisation/position.rb @@ -0,0 +1,28 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + class Position + def initialize(line:, character:) + raise ArgumentError, "line must be an integer" unless line.is_a?(Integer) + raise ArgumentError, "character must be an integer" unless character.is_a?(Integer) + + @line = line + @character = character + end + + attr_reader :line, :character + + def as_json(*) + { + "line" => line, + "character" => character, + } + end + + def to_json(*args) + as_json.to_json(*args) + end + end +end diff --git a/ruby/serialisation/range.rb b/ruby/serialisation/range.rb new file mode 100644 index 0000000..37c3665 --- /dev/null +++ b/ruby/serialisation/range.rb @@ -0,0 +1,28 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + class Range + def initialize(start_pos:, end_pos:) + raise ArgumentError, "start_pos must be a Pos" unless start_pos.is_a?(Serialisation::Pos) + raise ArgumentError, "end_pos must be a Pos" unless end_pos.is_a?(Serialisation::Pos) + + @start_pos = start_pos + @end_pos = end_pos + end + + attr_reader :start_pos, :end_pos + + def as_json(*) + { + "start" => start_pos.as_json, + "end" => end_pos.as_json, + } + end + + def to_json(*args) + as_json.to_json(*args) + end + end +end diff --git a/ruby/serialisation/status/base.rb b/ruby/serialisation/status/base.rb new file mode 100644 index 0000000..7ca2fd0 --- /dev/null +++ b/ruby/serialisation/status/base.rb @@ -0,0 +1,44 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Base + def initialize(result:, test:, duration: nil, message: []) + raise ArgumentError, "test must be a TestItem" unless test.is_a?(Serialisation::TestItem) + if duration + raise ArgumentError, "duration must be a number" unless duration.is_a?(Numeric) + end + if message + raise ArgumentError, "message must be an array of TestMessage" unless message.is_a?(Array) + raise ArgumentError, "message must be an array of TestMessage" unless message.all?{ |m| m.is_a?(Serialisation::Status::TestMessage) } + end + + @result = result + @test = test + @duration = duration + @message = message + end + + attr_reader :result, :test, :duration, :message + + def json_keys + raise "Not implemented" + end + + def as_json(*) + { + "result" => result, + "test" => test.as_json, + "duration" => duration, + "message" => message.map(&:as_json), + }.slice(*json_keys) + end + + def to_json(*args) + as_json.to_json(*args) + end + end + end +end diff --git a/ruby/serialisation/status/enqueued.rb b/ruby/serialisation/status/enqueued.rb new file mode 100644 index 0000000..f66b96e --- /dev/null +++ b/ruby/serialisation/status/enqueued.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Enqueued < Base + KEYS = %w[result test].freeze + + def initialize(test:) + super(result: "enqueued", test: test) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/errored.rb b/ruby/serialisation/status/errored.rb new file mode 100644 index 0000000..a7da564 --- /dev/null +++ b/ruby/serialisation/status/errored.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Errored < Base + KEYS = %w[result test duration message].freeze + + def initialize(test:, message:, duration: nil) + super(result: "errored", test: test, message: message, duration: duration) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/failed.rb b/ruby/serialisation/status/failed.rb new file mode 100644 index 0000000..7ceb8ca --- /dev/null +++ b/ruby/serialisation/status/failed.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Failed < Base + KEYS = %w[result test duration message].freeze + + def initialize(test:, message:, duration: nil) + super(result: "failed", test: test, message: message, duration: duration) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/passed.rb b/ruby/serialisation/status/passed.rb new file mode 100644 index 0000000..502b41d --- /dev/null +++ b/ruby/serialisation/status/passed.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Passed < Base + KEYS = %w[result test duration].freeze + + def initialize(test:, duration: nil) + super(result: "passed", test: test, duration: duration) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/skipped.rb b/ruby/serialisation/status/skipped.rb new file mode 100644 index 0000000..04b9bb2 --- /dev/null +++ b/ruby/serialisation/status/skipped.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Skipped < Base + KEYS = %w[result test].freeze + + def initialize(test:) + super(result: "skipped", test: test) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/started.rb b/ruby/serialisation/status/started.rb new file mode 100644 index 0000000..3657cb4 --- /dev/null +++ b/ruby/serialisation/status/started.rb @@ -0,0 +1,19 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class Started < Base + KEYS = %w[result test].freeze + + def initialize(test:) + super(result: "started", test: test) + end + + def json_keys + KEYS + end + end + end +end diff --git a/ruby/serialisation/status/test_message.rb b/ruby/serialisation/status/test_message.rb new file mode 100644 index 0000000..931e66a --- /dev/null +++ b/ruby/serialisation/status/test_message.rb @@ -0,0 +1,42 @@ +# # frozen_string_literal: true + +require "json" + +module Serialisation + module Status + class TestMessage + def initialize(message:, location: nil, actual_output: nil, expected_output: nil) + raise ArgumentError, "message must be a String" unless message.is_a?(String) + if (location) + raise ArgumentError, "location must be a Location" unless location.is_a?(Serialisation::Location) + end + if (actual_output) + raise ArgumentError, "actual_output must be a String" unless actual_output.is_a?(String) + end + if (expected_output) + raise ArgumentError, "expected_output must be a String" unless expected_output.is_a?(String) + end + + @message = message + @location = location + @actual_output = actual_output + @expected_output = expected_output + end + + attr_reader :message, :location, :actual_output, :expected_output + + def as_json(*) + { + "message" => message, + "location" => location&.as_json, + "actualOutput" => actual_output, + "expectedOutput" => expected_output, + } + end + + def to_json(*args) + as_json.to_json(*args) + end + end + end +end diff --git a/ruby/serialisation/test_item.rb b/ruby/serialisation/test_item.rb new file mode 100644 index 0000000..9fd6dee --- /dev/null +++ b/ruby/serialisation/test_item.rb @@ -0,0 +1,63 @@ +# # frozen_string_literal: true + +require "json" +require "uri" + +module Serialisation + class TestItem + def initialize(id:, label:, uri: , range:, + description: nil, sort_text: nil, error: nil, tags: [], children: []) + raise ArgumentError, "id must be a String" unless id.is_a?(String) + raise ArgumentError, "label must be a String" unless label.is_a?(String) + raise ArgumentError, "uri must be a URI" unless uri.is_a?(URI::Generic) + raise ArgumentError, "range must be a Range" unless range.is_a?(Serialisation::Range) + if description + raise ArgumentError, "description must be a String" unless description.is_a?(String) + end + if sort_text + raise ArgumentError, "sort_text must be a String" unless sort_text.is_a?(String) + end + if error + raise ArgumentError, "error must be a String" unless error.is_a?(String) + end + if tags + raise ArgumentError, "tags must be an array of Strings" unless tags.is_a?(Array) + raise ArgumentError, "tags must be an array of Strings" unless tags.all?{ |t| t.is_a?(String) } + end + if children + raise ArgumentError, "children must be an array of TestItems" unless children.is_a?(Array) + raise ArgumentError, "children must be an array of TestItems" unless children.all?{ |c| c.is_a?(TestItem) } + end + + @id = id + @label = label + @uri = uri + @range = range + @description = description + @sort_text = sort_text + @error = error + @tags = tags + @children = children + end + + attr_reader :id, :label, :uri, :range, :description, :sort_text, :error, :tags, :children + + def as_json(*) + { + "id" => id, + "label" => label, + "uri" => uri.to_s, + "range" => range.as_json, + "description" => description, + "sortText" => sort_text, + "error" => error, + "tags" => tags, + "children" => children.map(&:as_json), + } + end + + def to_json(*args) + as_json.to_json(*args) + end + end +end diff --git a/ruby/serialisation/test_schema.json b/ruby/serialisation/test_schema.json index 0198b8c..719528f 100644 --- a/ruby/serialisation/test_schema.json +++ b/ruby/serialisation/test_schema.json @@ -2,6 +2,7 @@ "definitions": { "testItem": { "type": "object", + "description": "An item to be shown in the 'test explorer' view. Can represent either a test suite or a test itself", "required": [ "id", "label", @@ -35,51 +36,83 @@ }, "range": { "$ref": "#/definitions/range" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/definitions/testItem" + } } }, "additionalProperties": false }, - "testGroup": { + "testStatus": { "type": "object", "required": [ - "id", - "label", - "children", - "range" + "test", + "result" ], "properties": { - "id": { - "$ref": "#/definitions/testId" + "test": { + "$ref": "#/definitions/testItem" }, - "description": { - "$ref": "#/definitions/testDescription" + "duration": { + "description": "How long the test took to execute, in milliseconds", + "type": "number" }, - "label": { - "$ref": "#/definitions/testLabel" - }, - "sortText": { - "$ref": "#/definitions/testSortText" - }, - "tags": { + "message": { "type": "array", "items": { - "$ref": "#/definitions/testTag" + "$ref": "#/definitions/testMessage" } }, - "uri": { - "$ref": "#/definitions/testUri" + "result": { + "type": "string", + "description": "The result of running a test. If 'errored' or 'failed', it must have at least one message", + "enum": ["enqueued", "errored", "failed", "passed", "skipped", "started"] + } + }, + "additionalProperties": false + }, + "testMessage": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string", + "description": "Human readable message text to display. Can use Markdown formatting" }, - "error": { - "$ref": "#/definitions/testError" + "location": { + "$ref": "#/definitions/location", + "description": "The location associated with the message" }, - "children": { - "type": "array", - "items": { - "$ref": "#/definitions/testItem" - } + "actualOutput": { + "type": "string", + "description": "Actual test output. If given with expectedOutput, a diff view will be shown" }, + "expectedOutput": { + "type": "string", + "description": "Expected test output. If given with actualOutput, a diff view will be shown" + }, + "additionalProperties": false + } + }, + "location": { + "type": "object", + "required": [ + "uri", + "range" + ], + "properties": { "range": { - "$ref": "#/definitions/range" + "$ref": "#/definitions/range", + "description": "The document range of this location" + }, + "uri": { + "$ref": "#/definitions/testUri", + "description": "The resource identifier of this location" } }, "additionalProperties": false From 8346fe4bad895f368ea38038ba76fdaed2611037 Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Mon, 8 May 2023 22:56:04 +0100 Subject: [PATCH 3/6] Use new serialisation with Minitest::List --- ruby/serialisation/position.rb | 2 +- ruby/serialisation/range.rb | 4 +-- ruby/vscode/minitest.rb | 6 ++++- ruby/vscode/minitest/tests.rb | 49 ++++++++++++++++++++++++---------- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/ruby/serialisation/position.rb b/ruby/serialisation/position.rb index ecebd22..0d9793a 100644 --- a/ruby/serialisation/position.rb +++ b/ruby/serialisation/position.rb @@ -4,7 +4,7 @@ module Serialisation class Position - def initialize(line:, character:) + def initialize(line:, character: 0) raise ArgumentError, "line must be an integer" unless line.is_a?(Integer) raise ArgumentError, "character must be an integer" unless character.is_a?(Integer) diff --git a/ruby/serialisation/range.rb b/ruby/serialisation/range.rb index 37c3665..37c067b 100644 --- a/ruby/serialisation/range.rb +++ b/ruby/serialisation/range.rb @@ -5,8 +5,8 @@ module Serialisation class Range def initialize(start_pos:, end_pos:) - raise ArgumentError, "start_pos must be a Pos" unless start_pos.is_a?(Serialisation::Pos) - raise ArgumentError, "end_pos must be a Pos" unless end_pos.is_a?(Serialisation::Pos) + raise ArgumentError, "start_pos must be a Position" unless start_pos.is_a?(Serialisation::Position) + raise ArgumentError, "end_pos must be a Position" unless end_pos.is_a?(Serialisation::Position) @start_pos = start_pos @end_pos = end_pos diff --git a/ruby/vscode/minitest.rb b/ruby/vscode/minitest.rb index 093c902..8a0b3eb 100644 --- a/ruby/vscode/minitest.rb +++ b/ruby/vscode/minitest.rb @@ -22,7 +22,11 @@ module Minitest def list(io = $stdout) io.sync = true if io.respond_to?(:"sync=") - data = { version: ::Minitest::VERSION, examples: tests.all } + data = { + runner: "minitest", + version: ::Minitest::VERSION, + items: tests.all.map(&:as_json) + } json = ENV.key?("PRETTY") ? JSON.pretty_generate(data) : JSON.generate(data) io.puts "START_OF_TEST_JSON#{json}END_OF_TEST_JSON" end diff --git a/ruby/vscode/minitest/tests.rb b/ruby/vscode/minitest/tests.rb index ce84dbf..7384a42 100644 --- a/ruby/vscode/minitest/tests.rb +++ b/ruby/vscode/minitest/tests.rb @@ -1,4 +1,8 @@ require "rake" +require "uri" +require "serialisation/position" +require "serialisation/range" +require "serialisation/test_item" module VSCode module Minitest @@ -34,30 +38,47 @@ def build_list tests = [] ::Minitest::Runnable.runnables.map do |runnable| + file_name = nil + puts "runnable #{runnable.name}\n" file_tests = runnable.runnable_methods.map do |test_name| + puts "test #{test_name}\n" path, line = runnable.instance_method(test_name).source_location + unless file_name + index = path.rindex(/[\\\/]/) + file_name = path.slice(index + 1, path.length - index) + end full_path = File.expand_path(path, VSCode.project_root) path = full_path.gsub(VSCode.project_root.to_s, ".") path = "./#{path}" unless path.match?(/^\./) description = test_name.gsub(/^test_[:\s]*/, "") description = description.tr("_", " ") unless description.match?(/\s/) - { - description: description, - full_description: description, - file_path: path, - full_path: full_path, - line_number: line, - klass: runnable.name, - method: test_name, - runnable: runnable - } + puts "end\n" + ::Serialisation::TestItem.new( + id: test_name, + label: description, + uri: URI.parse("file:///#{full_path}"), + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: line), + end_pos: ::Serialisation::Position.new(line: line), + ), + sort_text: line.to_s + ) end - file_tests.sort_by! { |t| t[:line_number] } - file_tests.each do |t| - t[:id] = "#{t[:file_path]}[#{t[:line_number]}]" + + unless file_tests.length.zero? + file_item = ::Serialisation::TestItem.new( + id: file_name, + label: file_name, + uri: file_tests.first.uri, + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: 0), + end_pos: ::Serialisation::Position.new(line: 0), + ), + children: file_tests, + ) + tests << file_item end - tests.concat(file_tests) end tests end From c1e1092ab65db37a1bfbd7d0a8c4a6d09317673e Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Fri, 2 Jun 2023 10:27:44 +0100 Subject: [PATCH 4/6] Use new serialisation format for all Minitest reporting --- ruby/serialisation/range.rb | 4 +- ruby/serialisation/status/test_message.rb | 3 +- ruby/serialisation/test_item.rb | 3 +- ruby/test/minitest/rake_task_test.rb | 153 ++++++++++++---------- ruby/vscode/minitest.rb | 15 ++- ruby/vscode/minitest/reporter.rb | 92 ++++++++----- ruby/vscode/minitest/tests.rb | 49 ++----- 7 files changed, 174 insertions(+), 145 deletions(-) diff --git a/ruby/serialisation/range.rb b/ruby/serialisation/range.rb index 37c067b..9fc63f5 100644 --- a/ruby/serialisation/range.rb +++ b/ruby/serialisation/range.rb @@ -4,8 +4,10 @@ module Serialisation class Range - def initialize(start_pos:, end_pos:) + def initialize(start_pos:, end_pos: nil) raise ArgumentError, "start_pos must be a Position" unless start_pos.is_a?(Serialisation::Position) + + end_pos = start_pos if end_pos.nil? raise ArgumentError, "end_pos must be a Position" unless end_pos.is_a?(Serialisation::Position) @start_pos = start_pos diff --git a/ruby/serialisation/status/test_message.rb b/ruby/serialisation/status/test_message.rb index 931e66a..636fdbd 100644 --- a/ruby/serialisation/status/test_message.rb +++ b/ruby/serialisation/status/test_message.rb @@ -23,7 +23,8 @@ def initialize(message:, location: nil, actual_output: nil, expected_output: nil @expected_output = expected_output end - attr_reader :message, :location, :actual_output, :expected_output + attr_reader :message, :location + attr_accessor :actual_output, :expected_output def as_json(*) { diff --git a/ruby/serialisation/test_item.rb b/ruby/serialisation/test_item.rb index 9fd6dee..54ab9ae 100644 --- a/ruby/serialisation/test_item.rb +++ b/ruby/serialisation/test_item.rb @@ -40,7 +40,8 @@ def initialize(id:, label:, uri: , range:, @children = children end - attr_reader :id, :label, :uri, :range, :description, :sort_text, :error, :tags, :children + attr_reader :id, :label, :uri, :range, :description, :sort_text, :tags, :children + attr_accessor :error def as_json(*) { diff --git a/ruby/test/minitest/rake_task_test.rb b/ruby/test/minitest/rake_task_test.rb index 8e4788c..952bc20 100644 --- a/ruby/test/minitest/rake_task_test.rb +++ b/ruby/test/minitest/rake_task_test.rb @@ -74,9 +74,11 @@ def env end def test_test_list - stdout, _, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:list", chdir: dir.to_s) + stdout, stderr, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:list", chdir: dir.to_s) + assert_equal "", stderr assert_predicate status, :success? + assert_match(/START_OF_TEST_JSON(.*)END_OF_TEST_JSON/, stdout) stdout =~ /START_OF_TEST_JSON(.*)END_OF_TEST_JSON/ @@ -84,58 +86,67 @@ def test_test_list [ { - description: "square of one", - full_description: "square of one", - file_path: "./test/square_test.rb", - full_path: (dir + "test/square_test.rb").to_s, - line_number: 4, - klass: "SquareTest", - method: "test_square_of_one", - runnable: "SquareTest", - id: "./test/square_test.rb[4]" + id: "./test/square_test.rb[4]", + label: "square of one", + range: { + start: { line: 3, character: 0 }, + end: { line: 3, character: 0 } + }, + description: nil, + sortText: "4", + error: nil, + tags: [], + children: [] }, { - description: "square of two", - full_description: "square of two", - file_path: "./test/square_test.rb", - full_path: (dir + "test/square_test.rb").to_s, - line_number: 8, - klass: "SquareTest", - method: "test_square_of_two", - runnable: "SquareTest", - id: "./test/square_test.rb[8]" + id: "./test/square_test.rb[8]", + label: "square of two", + range: { + start: { line: 7, character: 0 }, + end: { line: 7, character: 0 } + }, + description: nil, + sortText: "8", + error: nil, + tags: [], + children: [] }, { - description: "square error", - full_description: "square error", - file_path: "./test/square_test.rb", - full_path: (dir + "test/square_test.rb").to_s, - line_number: 12, - klass: "SquareTest", - method: "test_square_error", - runnable: "SquareTest", - id: "./test/square_test.rb[12]" + id: "./test/square_test.rb[12]", + label: "square error", + range: { + start: { line: 11, character: 0 }, + end: { line: 11, character: 0 } + }, + description: nil, + sortText: "12", + error: nil, + tags: [], + children: [] }, { - description: "square skip", - full_description: "square skip", - file_path: "./test/square_test.rb", - full_path: (dir + "test/square_test.rb").to_s, - line_number: 16, - klass: "SquareTest", - method: "test_square_skip", - runnable: "SquareTest", - id: "./test/square_test.rb[16]" + id: "./test/square_test.rb[16]", + label: "square skip", + range: { + start: { line: 15, character: 0 }, + end: { line: 15, character: 0 } + }, + description: nil, + sortText: "16", + error: nil, + tags: [], + children: [] } ].each do |expectation| - assert_includes(json[:examples], expectation) + assert_includes(json[:examples].map { |e| e.except(:uri) }, expectation) end end def test_test_run_all - stdout, _, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test", chdir: dir.to_s) + stdout, stderr, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test", chdir: dir.to_s) refute_predicate status, :success? + assert_equal "", stderr assert_match(/START_OF_TEST_JSON(.*)END_OF_TEST_JSON/, stdout) stdout =~ /START_OF_TEST_JSON(.*)END_OF_TEST_JSON/ @@ -146,48 +157,47 @@ def test_test_run_all assert_equal 4, examples.size assert_any(examples, pass_count: 1) do |example| - assert_equal "square error", example[:description] - assert_equal "errored", example[:status] - assert_nil example[:pending_message] - refute_nil example[:exception] - assert_equal "Minitest::UnexpectedError", example.dig(:exception, :class) - assert_match(/RuntimeError:/, example.dig(:exception, :message)) - assert_instance_of Array, example.dig(:exception, :backtrace) - assert_instance_of Array, example.dig(:exception, :full_backtrace) - assert_equal 12, example.dig(:exception, :position) + assert_equal "errored", example[:result] + assert_equal "square error", example.dig(:test, :label) + assert_nil example.dig(:test, :error) + refute_nil example[:message] + # assert_equal "Minitest::UnexpectedError", example.dig(:exception, :class) + assert_match(/RuntimeError:/, example.dig(:message, 0, :message)) + assert_equal 11, example.dig(:message, 0, :location, :range, :start, :line) end assert_any(examples, pass_count: 1) do |example| - assert_equal "square of one", example[:description] - assert_equal "passed", example[:status] - assert_nil example[:pending_message] - assert_nil example[:exception] + assert_equal "passed", example[:result] + assert_equal "square of one", example.dig(:test, :label) + assert_nil example.dig(:test, :error) + assert_nil example[:message] end assert_any(examples, pass_count: 1) do |example| - assert_equal "square of two", example[:description] - assert_equal "failed", example[:status] - assert_nil example[:pending_message] - refute_nil example[:exception] - assert_equal "Minitest::Assertion", example.dig(:exception, :class) - assert_equal "Expected: 3\n Actual: 4", example.dig(:exception, :message) - assert_instance_of Array, example.dig(:exception, :backtrace) - assert_instance_of Array, example.dig(:exception, :full_backtrace) - assert_equal 8, example.dig(:exception, :position) + assert_equal "failed", example[:result] + assert_equal "square of two", example.dig(:test, :label) + assert_nil example.dig(:test, :error) + refute_nil example[:message] + # assert_equal "Minitest::Assertion", example.dig(:exception, :class) + assert_match /Expected: 3\n Actual: 4/, example.dig(:message, 0, :message) + assert_equal "3", example.dig(:message, 0, :expectedOutput) + assert_equal "4", example.dig(:message, 0, :actualOutput) + assert_equal 7, example.dig(:message, 0, :location, :range, :start, :line) end assert_any(examples, pass_count: 1) do |example| - assert_equal "square skip", example[:description] - assert_equal "skipped", example[:status] - assert_equal "This is skip", example[:pending_message] - assert_nil example[:exception] + assert_equal "skipped", example[:result] + assert_equal "square skip", example.dig(:test, :label) + assert_equal "This is skip", example.dig(:test, :error) + assert_nil example[:message] end end def test_test_run_file - stdout, _, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test/square_test.rb", chdir: dir.to_s) + stdout, stderr, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test/square_test.rb", chdir: dir.to_s) refute_predicate status, :success? + assert_equal "", stderr assert_match(/START_OF_TEST_JSON(.*)END_OF_TEST_JSON/, stdout) stdout =~ /START_OF_TEST_JSON(.*)END_OF_TEST_JSON/ @@ -199,9 +209,10 @@ def test_test_run_file end def test_test_run_file_line - stdout, _, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test/square_test.rb:4 test/square_test.rb:16", chdir: dir.to_s) + stdout, stderr, status = Open3.capture3(env, "rake -R #{__dir__}/../.. vscode:minitest:run test/square_test.rb:4 test/square_test.rb:16", chdir: dir.to_s) assert_predicate status, :success? + assert_equal "", stderr assert_match(/START_OF_TEST_JSON(.*)END_OF_TEST_JSON/, stdout) stdout =~ /START_OF_TEST_JSON(.*)END_OF_TEST_JSON/ @@ -212,13 +223,13 @@ def test_test_run_file_line assert_equal 2, examples.size assert_any(examples, pass_count: 1) do |example| - assert_equal "square of one", example[:description] - assert_equal "passed", example[:status] + assert_equal "square of one", example[:test][:label] + assert_equal "passed", example[:result] end assert_any(examples, pass_count: 1) do |example| - assert_equal "square skip", example[:description] - assert_equal "skipped", example[:status] + assert_equal "square skip", example[:test][:label] + assert_equal "skipped", example[:result] end end end diff --git a/ruby/vscode/minitest.rb b/ruby/vscode/minitest.rb index 8a0b3eb..81da104 100644 --- a/ruby/vscode/minitest.rb +++ b/ruby/vscode/minitest.rb @@ -17,6 +17,19 @@ def project_root @project_root ||= Pathname.new(Dir.pwd) end + def test_item(test) + line = test[:line_number] || 1 + ::Serialisation::TestItem.new( + id: test[:id], + label: test[:description], + uri: URI.parse("file:///#{test[:full_path]}"), + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: line - 1), + ), + sort_text: (test[:line_number] || test[:id]).to_s + ) + end + module Minitest module_function @@ -25,7 +38,7 @@ def list(io = $stdout) data = { runner: "minitest", version: ::Minitest::VERSION, - items: tests.all.map(&:as_json) + examples: tests.all.map { |t| VSCode.test_item(t).as_json } } json = ENV.key?("PRETTY") ? JSON.pretty_generate(data) : JSON.generate(data) io.puts "START_OF_TEST_JSON#{json}END_OF_TEST_JSON" diff --git a/ruby/vscode/minitest/reporter.rb b/ruby/vscode/minitest/reporter.rb index 0990740..697b1bb 100644 --- a/ruby/vscode/minitest/reporter.rb +++ b/ruby/vscode/minitest/reporter.rb @@ -1,10 +1,26 @@ # frozen_string_literal: true +require "serialisation/location" +require "serialisation/position" +require "serialisation/range" +require "serialisation/status/base" +require "serialisation/status/enqueued" +require "serialisation/status/errored" +require "serialisation/status/failed" +require "serialisation/status/passed" +require "serialisation/status/skipped" +require "serialisation/status/started" +require "serialisation/status/test_message" +require "serialisation/test_item" +require "uri" + module VSCode module Minitest class Reporter < ::Minitest::Reporter attr_accessor :assertions, :count, :results, :start_time, :total_time, :failures, :errors, :skips + ASSERTION_REGEX = /(?:(?.*)\.)?\s*Expected:? (?.*)\s*(?:Actual:|to be) (?.*)/.freeze + def initialize(io = $stdout, options = {}) super io.sync = true if io.respond_to?(:"sync=") @@ -19,16 +35,16 @@ def start def prerecord(klass, meth) data = VSCode::Minitest.tests.find_by(klass: klass.to_s, method: meth) - io.puts "\nRUNNING: #{data[:id]}\n" + io.puts "#{::Serialisation::Status::Started.new(test: VSCode.test_item(data)).to_json}\n" end def record(result) self.count += 1 self.assertions += result.assertions results << result - data = vscode_result(result, false) + data = vscode_result(result) - io.puts "#{data[:status]}#{data[:exception]}: #{data[:id]}\n" + io.puts "#{data.to_json}\n" end def report @@ -58,48 +74,46 @@ def vscode_data }, summary_line: "Total time: #{total_time}, Runs: #{count}, Assertions: #{assertions}, " \ "Failures: #{failures}, Errors: #{errors}, Skips: #{skips}", - examples: results.map { |r| vscode_result(r, true) } + examples: results.map { |r| vscode_result(r).as_json } } end - def vscode_result(result, is_report) - base = VSCode::Minitest.tests.find_by(klass: result.klass, method: result.name).dup - - base[:status] = vscode_status(result, is_report) - base[:pending_message] = result.skipped? ? result.failure.message : nil - base[:exception] = vscode_exception(result, base, is_report) - base[:duration] = result.time - base.compact - end - - def vscode_status(result, is_report) + def vscode_result(result) + data = VSCode::Minitest.tests.find_by(klass: result.klass, method: result.name).dup + test = VSCode.test_item(data) if result.skipped? - status = 'skipped' + # Not sure if there's a better place to put this message + test.error = result.failure.message + return ::Serialisation::Status::Skipped.new(test: test) elsif result.passed? - status = 'passed' + return ::Serialisation::Status::Passed.new(test: test, duration: result.time) else - e = result.failure.exception - status = e.class.name == ::Minitest::UnexpectedError.name ? 'errored' : 'failed' + msg = [vscode_test_message(result, data)] + if result.failure.exception.class.name == ::Minitest::UnexpectedError.name + return ::Serialisation::Status::Errored.new(test: test, message: msg, duration: result.time) + else + return ::Serialisation::Status::Failed.new(test: test, message: msg, duration: result.time) + end end - is_report ? status : status.upcase end - def vscode_exception(result, data, is_report) + def vscode_test_message(result, data) return if result.passed? || result.skipped? err = result.failure.exception backtrace = expand_backtrace(err.backtrace) - if is_report - { - class: err.class.name, - message: err.message, - backtrace: clean_backtrace(backtrace), - full_backtrace: backtrace, - position: exception_position(backtrace, data[:full_path]) || data[:line_number] - } - else - "(#{err.class.name}:#{err.message.tr("\n", ' ').strip})" + msg = ::Serialisation::Status::TestMessage.new( + message: "#{err.message}\n#{clean_backtrace(backtrace).join("\n")}", + location: exception_location(backtrace, data) + ) + + diff_match = err.message.match(ASSERTION_REGEX) + if diff_match + msg.expected_output = diff_match[:exp] + msg.actual_output = diff_match[:act] end + + msg end def expand_backtrace(backtrace) @@ -121,11 +135,19 @@ def clean_backtrace(backtrace) end end - def exception_position(backtrace, file) - line = backtrace.find { |frame| frame.start_with?(file) } - return unless line + def exception_location(backtrace, data) + frame = backtrace.find { |frame| frame.start_with?(data[:full_path]) } + if frame + path, line = frame.split(':') + else + path = data[:full_path] + line = data[:line_number] ? data[:line_number] : 1 + end - line.split(':')[1].to_i + ::Serialisation::Location.new( + uri: URI.parse("file:///#{path}"), + range: ::Serialisation::Range.new(start_pos: ::Serialisation::Position.new(line: line - 1)), + ) end end end diff --git a/ruby/vscode/minitest/tests.rb b/ruby/vscode/minitest/tests.rb index 7384a42..ce84dbf 100644 --- a/ruby/vscode/minitest/tests.rb +++ b/ruby/vscode/minitest/tests.rb @@ -1,8 +1,4 @@ require "rake" -require "uri" -require "serialisation/position" -require "serialisation/range" -require "serialisation/test_item" module VSCode module Minitest @@ -38,47 +34,30 @@ def build_list tests = [] ::Minitest::Runnable.runnables.map do |runnable| - file_name = nil - puts "runnable #{runnable.name}\n" file_tests = runnable.runnable_methods.map do |test_name| - puts "test #{test_name}\n" path, line = runnable.instance_method(test_name).source_location - unless file_name - index = path.rindex(/[\\\/]/) - file_name = path.slice(index + 1, path.length - index) - end full_path = File.expand_path(path, VSCode.project_root) path = full_path.gsub(VSCode.project_root.to_s, ".") path = "./#{path}" unless path.match?(/^\./) description = test_name.gsub(/^test_[:\s]*/, "") description = description.tr("_", " ") unless description.match?(/\s/) - puts "end\n" - ::Serialisation::TestItem.new( - id: test_name, - label: description, - uri: URI.parse("file:///#{full_path}"), - range: ::Serialisation::Range.new( - start_pos: ::Serialisation::Position.new(line: line), - end_pos: ::Serialisation::Position.new(line: line), - ), - sort_text: line.to_s - ) + { + description: description, + full_description: description, + file_path: path, + full_path: full_path, + line_number: line, + klass: runnable.name, + method: test_name, + runnable: runnable + } end - - unless file_tests.length.zero? - file_item = ::Serialisation::TestItem.new( - id: file_name, - label: file_name, - uri: file_tests.first.uri, - range: ::Serialisation::Range.new( - start_pos: ::Serialisation::Position.new(line: 0), - end_pos: ::Serialisation::Position.new(line: 0), - ), - children: file_tests, - ) - tests << file_item + file_tests.sort_by! { |t| t[:line_number] } + file_tests.each do |t| + t[:id] = "#{t[:file_path]}[#{t[:line_number]}]" end + tests.concat(file_tests) end tests end From d3b1386fd425299fabb5eaa537360492d3e33ff2 Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Thu, 8 Jun 2023 21:23:16 +0100 Subject: [PATCH 5/6] Use new serialisation format for all RSpec reporting --- ruby/custom_formatter.rb | 420 +++++++++++++++++++++++++----- ruby/rspecs/contexts_spec.rb | 65 +++++ ruby/rspecs/folder/folder_spec.rb | 9 + ruby/serialisation/test_item.rb | 6 +- ruby/vscode/minitest/reporter.rb | 1 + 5 files changed, 434 insertions(+), 67 deletions(-) create mode 100644 ruby/rspecs/contexts_spec.rb create mode 100644 ruby/rspecs/folder/folder_spec.rb diff --git a/ruby/custom_formatter.rb b/ruby/custom_formatter.rb index 8dcd33c..cee09a6 100644 --- a/ruby/custom_formatter.rb +++ b/ruby/custom_formatter.rb @@ -1,18 +1,32 @@ # frozen_string_literal: true +require 'json' require 'rspec/core' require 'rspec/core/formatters/base_formatter' -require 'json' +# require 'rspec/expectations' +require_relative "serialisation/location" +require_relative "serialisation/position" +require_relative "serialisation/range" +require_relative "serialisation/status/base" +require_relative "serialisation/status/enqueued" +require_relative "serialisation/status/errored" +require_relative "serialisation/status/failed" +require_relative "serialisation/status/passed" +require_relative "serialisation/status/skipped" +require_relative "serialisation/status/started" +require_relative "serialisation/status/test_message" +require_relative "serialisation/test_item" # Formatter to emit RSpec test status information in the required format for the extension class CustomFormatter < RSpec::Core::Formatters::BaseFormatter RSpec::Core::Formatters.register self, :message, + :deprecation, :dump_summary, :stop, :seed, :close, - # :example_group_started, + :example_group_started, :example_passed, :example_failed, :example_pending, @@ -22,6 +36,8 @@ class CustomFormatter < RSpec::Core::Formatters::BaseFormatter def initialize(output) super + @test_items = {} + @group_items = {} @output_hash = { version: RSpec::Core::Version::STRING } @@ -31,6 +47,16 @@ def message(notification) (@output_hash[:messages] ||= []) << notification.message end + def deprecation(notification) + # TODO + # DeprecationNotification methods: + # - call_site + # - deprecated/message + # - replacement + $stderr.puts "\ndeprecation:\n\tcall_site:#{notification.call_site}\n\tmessage:#{notification.message}\n\treplacement:#{notification.replacement}" + super(notification) + end + def dump_summary(summary) @output_hash[:summary] = { duration: summary.duration, @@ -43,19 +69,20 @@ def dump_summary(summary) end def stop(notification) - @output_hash[:examples] = notification.examples.map do |example| - format_example(example).tap do |hash| - e = example.exception - if e - hash[:exception] = { - class: e.class.name, - message: e.message, - backtrace: e.backtrace, - position: exception_position(e.backtrace_locations, example.metadata) - } - end - end - end + # @output_hash[:examples] = notification.examples.map do |example| + # format_example(example).tap do |hash| + # e = example.exception + # if e + # hash[:exception] = { + # class: e.class.name, + # message: e.message, + # backtrace: e.backtrace, + # position: exception_position(e.backtrace_locations, example.metadata) + # } + # end + # end + # end + @output_hash[:examples] = @test_items.values.map(&:as_json) end def seed(notification) @@ -69,82 +96,345 @@ def close(_notification) end def example_passed(notification) - output.write "PASSED: #{notification.example.id}\n" + # output.write "PASSED: #{notification.example.id}\n" + output.write "#{passed_status(notification.example).to_json}\n" end def example_failed(notification) - klass = notification.example.exception.class - status = exception_is_error?(klass) ? 'ERRORED' : 'FAILED' - exception_message = notification.example.exception.message.gsub(/\s+/, ' ').strip - output.write "#{status}(#{klass.name}:#{exception_message}): " \ - "#{notification.example.id}\n" - # This isn't exposed for simplicity, need to figure out how to handle this later. - # output.write "#{notification.exception.backtrace.to_json}\n" + # klass = notification.example.exception.class + # status = exception_is_error?(klass) ? 'ERRORED' : 'FAILED' + # exception_message = notification.example.exception.message.gsub(/\s+/, ' ').strip + # output.write "#{status}(#{klass.name}:#{exception_message}): " \ + # "#{notification.example.id}\n" + # # This isn't exposed for simplicity, need to figure out how to handle this later. + # # output.write "#{notification.exception.backtrace.to_json}\n" + output.write "#{failed_status(notification.example).to_json}\n" end def example_pending(notification) - output.write "SKIPPED: #{notification.example.id}\n" + # output.write "SKIPPED: #{notification.example.id}\n" + output.write "#{skipped_status(notification.example).to_json}\n" end def example_started(notification) - output.write "RUNNING: #{notification.example.id}\n" + # output.write "RUNNING: #{notification.example.id}\n" + # dump_notification(notification) + output.write "#{started_status(notification.example).to_json}\n" end - # def example_group_started(notification) - # output.write "RUNNING: #{notification.group.id}\n" - # end + def example_group_started(notification) + # output.write "RUNNING: #{notification.group.id}\n" + # dump_notification(notification) + + item = create_group_item(notification.group) + output.write "#{::Serialisation::Status::Started.new(test: item).to_json}\n" + end private # Properties of example: - # block - # description_args - # description - # full_description - # described_class - # file_path - # line_number - # location - # absolute_file_path - # rerun_file_path - # scoped_id - # type - # execution_result - # example_group - # shared_group_inclusion_backtrace - # last_run_status - def format_example(example) - { - id: example.id, - description: example.description, - full_description: example.full_description, - status: example_status(example), - file_path: example.metadata[:file_path], - line_number: example.metadata[:line_number], - type: example.metadata[:type], - pending_message: example.execution_result.pending_message, - duration: example.execution_result.run_time - } - end - def exception_position(backtrace, metadata) - location = backtrace&.find { |frame| frame.path.end_with?(metadata[:file_path]) } - return metadata[:line_number] unless location + # def format_example(example) + # # dump_example(example) + # { + # id: example.id, + # description: example.description, + # full_description: example.full_description, + # status: example_status(example), + # file_path: example.metadata[:file_path], + # line_number: example.metadata[:line_number], + # type: example.metadata[:type], + # pending_message: example.execution_result.pending_message, + # duration: example.execution_result.run_time + # } + # end + + def exception_location(backtrace, metadata) + frame = backtrace.find { |frame| frame.path.end_with?(metadata[:file_path]) } + if frame + line = frame.lineno + else + line = metadata[:line_number] ? metadata[:line_number] : 1 + end - location.lineno + ::Serialisation::Location.new( + uri: URI.parse("file:///#{metadata[:absolute_file_path]}"), + range: ::Serialisation::Range.new(start_pos: ::Serialisation::Position.new(line: line - 1)), + ) end def example_status(example) - if example.exception && exception_is_error?(example.exception.class) - 'errored' + if example.exception + failed_status(example) elsif example.execution_result.status == :pending - 'skipped' + skipped_status(example) else - example.execution_result.status.to_s + passed_status(example) end end + def started_status(example) + ::Serialisation::Status::Started.new(test: test_item(example)) + end + + def failed_status(example) + klass = exception_is_error?(example.exception.class) ? ::Serialisation::Status::Errored : ::Serialisation::Status::Failed + klass.new( + test: test_item(example), + message: test_message(example), + duration: example.execution_result.run_time, + ) + end + + def skipped_status(example) + item = test_item(example) + # Not sure if there's a better place to put this message + item.error = example.execution_result.pending_message, + ::Serialisation::Status::Skipped.new(test: item) + end + + def passed_status(example) + ::Serialisation::Status::Passed.new( + test: test_item(example), + duration: example.execution_result.run_time, + ) + end + def exception_is_error?(exception_class) !exception_class.to_s.start_with?('RSpec') end + + def multiple_exception_container?(exception) + exception.is_a? RSpec::Core::MultipleExceptionError + # || exception.is_a? RSpec::Expectations::MultipleExpectationsNotMetError + end + + def test_message(example) + if multiple_exception_container? example.exception + example.exception.all_exceptions.map do |sub_exception| + test_message_from_exception(example, sub_exception) + end + else + [test_message_from_exception(example, example.exception)] + end + end + + def test_message_from_exception(example, exception) + msg = ::Serialisation::Status::TestMessage.new( + message: "#{exception.message}\n#{exception.backtrace.join("\n")}", + location: exception_location(backtrace, example.metadata) + ) + + # diff_match = err.message.match(ASSERTION_REGEX) + # if diff_match + # msg.expected_output = diff_match[:exp] + # msg.actual_output = diff_match[:act] + # end + + msg + end + + def file_item(file_path, absolute_file_path) + return @test_items[file_path] if @test_items.key?(file_path) + + item = ::Serialisation::TestItem.new( + id: file_path, + label: file_path.split('/').last, + uri: URI.parse("file:///#{absolute_file_path}"), + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: 0), + ), + sort_text: file_path, + parent_ids: item_parents(file_path), + ) + @test_items[file_path] = item + item + end + + def create_group_item(group) + scoped_id = group.metadata[:scoped_id] + # puts "\ncreate_group_item - scoped_id: #{scoped_id}" + parent_item = file_item(group.file_path, group.metadata[:absolute_file_path]) + # puts "\ncreate_group_item - parent: #{parent_item.inspect}" + + if scoped_id != "1" + # Need to create child items for groups within file + # puts "\ncreate_group_item - creating new group item" + item = nil + + for i in 1..scoped_id.count(':') do + sub_id = "[#{scoped_id[0..(2 * i)]}]" + item = parent_item.children.find { |x| x.id.end_with?(sub_id)} + # puts "\ncreate_group_item - parent(#{sub_id}): #{item.inspect}" + + if item.nil? + item = ::Serialisation::TestItem.new( + id: group.id, + label: group.description, + uri: URI.parse("file:///#{group.metadata[:absolute_file_path]}"), + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: group.metadata[:line_number] - 1), + ), + sort_text: scoped_id + ) + # puts "\ncreate_group_item - parent(#{sub_id}): created #{item.inspect}" + parent_item.children << item + end + + parent_item = item + end + # puts "\ncreate_group_item - item: #{item.inspect}" + end + + example_group = group.example_group.to_s + anon_index = (example_group =~ /::Anonymous/) - 1 + example_group_name = example_group[0..anon_index] + # puts "\ncreate_group_item - example_group_name: #{example_group_name}" + @group_items[example_group_name] = parent_item + parent_item + end + + # def base_path(example) + # path_segments = example.file_path.split('/') + # if path_segments.first == '.' + # path_segments.shift + # end + # index = (example.absolute_file_path =~ /#{path_segments.first}/) - 1 + # example.absolute_file_path[0..index] + # end + + def item_parents(file_path) + path_segments = file_path.split('/') + if path_segments.first == '.' + path_segments.shift + end + path_segments[0..-2] + end + + # def shared_example?(example) + # example.metadata[:shared_group_inclusion_backtrace].length > 1 + # end + + def test_item(example) + # Standard metadata keys: (TODO - get tags by finding other keys) + # block + # description_args + # description + # full_description + # described_class + # file_path + # line_number + # location + # absolute_file_path + # rerun_file_path + # scoped_id + # type + # execution_result + # example_group + # shared_group_inclusion_backtrace + # last_run_status + + # puts "\nexample: #{example.inspect}" + + parent = @group_items[example.example_group.to_s] + # puts "\ntest_item - parent: #{parent.inspect} (#{example.example_group})" + + item = parent.children.select { |x| x.id == example.id }.first + # puts "\ntest_item - item: #{item.inspect}" + return item if item + + # puts "\ntest_item - creating new test item" + item = ::Serialisation::TestItem.new( + id: example.id, + label: example.description, + description: example.full_description, + uri: URI.parse("file:///#{example.metadata[:absolute_file_path]}"), + range: ::Serialisation::Range.new( + start_pos: ::Serialisation::Position.new(line: example.metadata[:line_number] - 1), + ), + sort_text: example.metadata[:scoped_id], + ) + # puts "\ntest_item - item: #{item.inspect}" + parent.children << item + item + end + + def dump_notification(notification) + if notification.respond_to?(:example) + dump_example(notification.example) + else + dump_example(notification.group, true) + end + end + + def dump_example(example, group = false) + $stderr.puts "\nexample#{group ? " group" : ""}:" + %i[ + id + class + described_class + description + execution_result + example_group + full_description + file_path + location + metadata + ].each do |prop| + if (example.respond_to? prop) + if prop == :execution_result + $stderr.puts "\n\t#{prop}:" + er = example.send(prop) + %i[ + exception + finished_at + pending_exception + pending_fixed + pending_message + run_time + started_at + status + ].each do |sub_prop| + $stderr.puts "\n\t\t#{sub_prop}: #{er.send(sub_prop)}" + end + elsif prop == :metadata + metadata = example.send(prop) + $stderr.puts "\n\t#{prop}: {" + dump_metadata(metadata) + $stderr.puts "\n\t}" + else + $stderr.puts "\n\t#{prop}: #{example.send(prop)}" + end + end + end + end + + def dump_metadata(metadata, indent = "\t\t") + %i[ + absolute_file_path + block + described_class + description_args + description + example_group + file_path + full_description + last_run_status + line_number + location + rerun_file_path + scoped_id + shared_group_inclusion_backtrace + ].each do |sub_prop| + if metadata.key? sub_prop + if sub_prop == :example_group + $stderr.puts "\n#{indent}#{sub_prop}: {" + dump_metadata(metadata[sub_prop], "\t\t\t") + $stderr.puts "\n#{indent}}" + else + $stderr.puts "\n#{indent}#{sub_prop}: #{metadata[sub_prop]}," + end + end + end + end end diff --git a/ruby/rspecs/contexts_spec.rb b/ruby/rspecs/contexts_spec.rb new file mode 100644 index 0000000..581c092 --- /dev/null +++ b/ruby/rspecs/contexts_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'rspec' + +describe 'Contexts' do + context 'when' do + context 'there' do + context 'are' do + context 'many' do + context 'levels' do + context 'of' do + context 'nested' do + context 'contexts' do + it "doesn't break the extension" do + expect('Hello text explorer!').to be_a(String) + end + end + end + end + end + end + + context 'fewer levels of nested contexts' do + it do + expect('Hello again text explorer!').to be_a(String) + end + end + end + end + end + + shared_examples_for "an even number" do + it "is divisible by 2" do + expect(value % 2).to be 0 + end + end + + shared_context "even number" do + let(:value) { 4 } + end + + shared_context "odd number" do + let(:value) { 5 } + end + + context 'with shared examples' do + shared_examples_for "an odd number" do + it "is not divisible by 2" do + expect(value % 2).to_not be 0 + end + end + + context 'when number is even' do + include_context "even number" + + it_behaves_like "an even number" + end + + context 'when number is odd' do + include_context "odd number" + + it_behaves_like "an odd number" + end + end +end diff --git a/ruby/rspecs/folder/folder_spec.rb b/ruby/rspecs/folder/folder_spec.rb new file mode 100644 index 0000000..cc4ef73 --- /dev/null +++ b/ruby/rspecs/folder/folder_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'rspec' + +describe 'Folder', :wobble do + it "wibble", tagged: true do + expect(1).to be 1 + end +end \ No newline at end of file diff --git a/ruby/serialisation/test_item.rb b/ruby/serialisation/test_item.rb index 54ab9ae..a3b8603 100644 --- a/ruby/serialisation/test_item.rb +++ b/ruby/serialisation/test_item.rb @@ -6,7 +6,7 @@ module Serialisation class TestItem def initialize(id:, label:, uri: , range:, - description: nil, sort_text: nil, error: nil, tags: [], children: []) + description: nil, sort_text: nil, error: nil, tags: [], children: [], parent_ids: []) raise ArgumentError, "id must be a String" unless id.is_a?(String) raise ArgumentError, "label must be a String" unless label.is_a?(String) raise ArgumentError, "uri must be a URI" unless uri.is_a?(URI::Generic) @@ -38,10 +38,11 @@ def initialize(id:, label:, uri: , range:, @error = error @tags = tags @children = children + @parent_ids = parent_ids end attr_reader :id, :label, :uri, :range, :description, :sort_text, :tags, :children - attr_accessor :error + attr_accessor :error, :parent_ids def as_json(*) { @@ -54,6 +55,7 @@ def as_json(*) "error" => error, "tags" => tags, "children" => children.map(&:as_json), + "parent_ids" => parent_ids, } end diff --git a/ruby/vscode/minitest/reporter.rb b/ruby/vscode/minitest/reporter.rb index 697b1bb..335e3cf 100644 --- a/ruby/vscode/minitest/reporter.rb +++ b/ruby/vscode/minitest/reporter.rb @@ -19,6 +19,7 @@ module Minitest class Reporter < ::Minitest::Reporter attr_accessor :assertions, :count, :results, :start_time, :total_time, :failures, :errors, :skips + # TODO: Hook Minitest::Assertions.things_to_diff to get the expected/actual values more reliably? ASSERTION_REGEX = /(?:(?.*)\.)?\s*Expected:? (?.*)\s*(?:Actual:|to be) (?.*)/.freeze def initialize(io = $stdout, options = {}) From 20bded8ee6dafbe68904d0633b6fc3f527b3bf58 Mon Sep 17 00:00:00 2001 From: Tabby Cromarty Date: Sun, 11 Jun 2023 13:08:18 +0100 Subject: [PATCH 6/6] Only send ID not full test in status updates (Not sure if right approach) --- ruby/custom_formatter.rb | 45 ++++++++++++++++++--------- ruby/serialisation/status/base.rb | 15 +++++---- ruby/serialisation/status/enqueued.rb | 6 ++-- ruby/serialisation/status/errored.rb | 6 ++-- ruby/serialisation/status/failed.rb | 6 ++-- ruby/serialisation/status/passed.rb | 6 ++-- ruby/serialisation/status/skipped.rb | 6 ++-- ruby/serialisation/status/started.rb | 6 ++-- 8 files changed, 57 insertions(+), 39 deletions(-) diff --git a/ruby/custom_formatter.rb b/ruby/custom_formatter.rb index cee09a6..2ce0be9 100644 --- a/ruby/custom_formatter.rb +++ b/ruby/custom_formatter.rb @@ -126,8 +126,7 @@ def example_group_started(notification) # output.write "RUNNING: #{notification.group.id}\n" # dump_notification(notification) - item = create_group_item(notification.group) - output.write "#{::Serialisation::Status::Started.new(test: item).to_json}\n" + output.write "#{group_started_status(notification.group).to_json}\n" end private @@ -165,37 +164,53 @@ def exception_location(backtrace, metadata) def example_status(example) if example.exception - failed_status(example) + failed_status(example, true) elsif example.execution_result.status == :pending - skipped_status(example) + skipped_status(example, true) else - passed_status(example) + passed_status(example, true) end end def started_status(example) - ::Serialisation::Status::Started.new(test: test_item(example)) + ::Serialisation::Status::Started.new(test_id: example.id) end - def failed_status(example) + def group_started_status(group) + ::Serialisation::Status::Started.new( + test_id: group.id, + test_item: create_group_item(group), + ) + end + + def failed_status(example, is_summary = false) klass = exception_is_error?(example.exception.class) ? ::Serialisation::Status::Errored : ::Serialisation::Status::Failed klass.new( - test: test_item(example), + test_id: example.id, + test_item: is_summary ? test_item(example) : nil, message: test_message(example), duration: example.execution_result.run_time, ) end - def skipped_status(example) - item = test_item(example) - # Not sure if there's a better place to put this message - item.error = example.execution_result.pending_message, - ::Serialisation::Status::Skipped.new(test: item) + def skipped_status(example, is_summary = false) + if is_summary + item = test_item(example) + # Not sure if there's a better place to put this message + item.error = example.execution_result.pending_message, + ::Serialisation::Status::Skipped.new( + test_id: example.id, + test_item: item, + ) + else + ::Serialisation::Status::Skipped.new(test_id: example.id) + end end - def passed_status(example) + def passed_status(example, is_summary = false) ::Serialisation::Status::Passed.new( - test: test_item(example), + test_id: example.id, + test_item: is_summary ? test_item(example) : nil, duration: example.execution_result.run_time, ) end diff --git a/ruby/serialisation/status/base.rb b/ruby/serialisation/status/base.rb index 7ca2fd0..a9a3c3b 100644 --- a/ruby/serialisation/status/base.rb +++ b/ruby/serialisation/status/base.rb @@ -5,8 +5,9 @@ module Serialisation module Status class Base - def initialize(result:, test:, duration: nil, message: []) - raise ArgumentError, "test must be a TestItem" unless test.is_a?(Serialisation::TestItem) + def initialize(result:, test_id:, test_item: nil, duration: nil, message: []) + raise ArgumentError, "test_id must be a String" unless test_id.is_a? String + raise ArgumentError, "test_item must be a TestItem" unless test_item.nil? || test_item.is_a?(Serialisation::TestItem) if duration raise ArgumentError, "duration must be a number" unless duration.is_a?(Numeric) end @@ -16,12 +17,13 @@ def initialize(result:, test:, duration: nil, message: []) end @result = result - @test = test + @test_id = test_id + @test_item = test_item @duration = duration @message = message end - attr_reader :result, :test, :duration, :message + attr_reader :result, :test_id, :test_item, :duration, :message def json_keys raise "Not implemented" @@ -30,10 +32,11 @@ def json_keys def as_json(*) { "result" => result, - "test" => test.as_json, + "test_id" => test_id, + "test_item" => test_item&.as_json || nil, "duration" => duration, "message" => message.map(&:as_json), - }.slice(*json_keys) + }.slice(*json_keys) end def to_json(*args) diff --git a/ruby/serialisation/status/enqueued.rb b/ruby/serialisation/status/enqueued.rb index f66b96e..9caf755 100644 --- a/ruby/serialisation/status/enqueued.rb +++ b/ruby/serialisation/status/enqueued.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Enqueued < Base - KEYS = %w[result test].freeze + KEYS = %w[result test_id test_item].freeze - def initialize(test:) - super(result: "enqueued", test: test) + def initialize(test_id:, test_item: nil) + super(result: "enqueued", test_id: test_id, test_item: test_item) end def json_keys diff --git a/ruby/serialisation/status/errored.rb b/ruby/serialisation/status/errored.rb index a7da564..5b7b348 100644 --- a/ruby/serialisation/status/errored.rb +++ b/ruby/serialisation/status/errored.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Errored < Base - KEYS = %w[result test duration message].freeze + KEYS = %w[result test_id test_item duration message].freeze - def initialize(test:, message:, duration: nil) - super(result: "errored", test: test, message: message, duration: duration) + def initialize(test_id:, message:, test_item: nil, duration: nil) + super(result: "errored", test_id: test_id, test_item: test_item, message: message, duration: duration) end def json_keys diff --git a/ruby/serialisation/status/failed.rb b/ruby/serialisation/status/failed.rb index 7ceb8ca..89dc553 100644 --- a/ruby/serialisation/status/failed.rb +++ b/ruby/serialisation/status/failed.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Failed < Base - KEYS = %w[result test duration message].freeze + KEYS = %w[result test_id test_item duration message].freeze - def initialize(test:, message:, duration: nil) - super(result: "failed", test: test, message: message, duration: duration) + def initialize(test_id:, message:, test_item: nil, duration: nil) + super(result: "failed", test_id: test_id, test_item: test_item, message: message, duration: duration) end def json_keys diff --git a/ruby/serialisation/status/passed.rb b/ruby/serialisation/status/passed.rb index 502b41d..80f9763 100644 --- a/ruby/serialisation/status/passed.rb +++ b/ruby/serialisation/status/passed.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Passed < Base - KEYS = %w[result test duration].freeze + KEYS = %w[result test_id test_item duration].freeze - def initialize(test:, duration: nil) - super(result: "passed", test: test, duration: duration) + def initialize(test_id:, test_item: nil, duration: nil) + super(result: "passed", test_id: test_id, test_item: test_item, duration: duration) end def json_keys diff --git a/ruby/serialisation/status/skipped.rb b/ruby/serialisation/status/skipped.rb index 04b9bb2..792c022 100644 --- a/ruby/serialisation/status/skipped.rb +++ b/ruby/serialisation/status/skipped.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Skipped < Base - KEYS = %w[result test].freeze + KEYS = %w[result test_id test_item].freeze - def initialize(test:) - super(result: "skipped", test: test) + def initialize(test_id:, test_item: nil) + super(result: "skipped", test_id: test_id, test_item: test_item) end def json_keys diff --git a/ruby/serialisation/status/started.rb b/ruby/serialisation/status/started.rb index 3657cb4..eb72119 100644 --- a/ruby/serialisation/status/started.rb +++ b/ruby/serialisation/status/started.rb @@ -5,10 +5,10 @@ module Serialisation module Status class Started < Base - KEYS = %w[result test].freeze + KEYS = %w[result test_id test_item].freeze - def initialize(test:) - super(result: "started", test: test) + def initialize(test_id:, test_item: nil) + super(result: "started", test_id: test_id, test_item: test_item) end def json_keys