From bf55cf83c88373744445171436d14735e1cf4c28 Mon Sep 17 00:00:00 2001 From: Luke Redmore Date: Sat, 8 Nov 2025 00:20:07 -0800 Subject: [PATCH 1/4] Support annotated strings in Gherkin lexer --- lib/rouge/demos/gherkin | 49 +++++++++++++++++++++++++++++++++++++ lib/rouge/lexers/gherkin.rb | 15 ++++++++++++ spec/lexers/gherkin_spec.rb | 33 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/lib/rouge/demos/gherkin b/lib/rouge/demos/gherkin index 78e4ba6e87..5e082db8a7 100644 --- a/lib/rouge/demos/gherkin +++ b/lib/rouge/demos/gherkin @@ -15,3 +15,52 @@ Feature: Addition | 20 | 30 | add | 50 | | 2 | 5 | add | 7 | | 0 | 40 | add | 40 | + + Scenario: Code samples + Given I have some code examples + """ruby + class Calculator + def multiply(x, y) + x * y + end + end + """ + And I have some code examples + """rust + fn safe_divide(x: f64, y: f64) -> Result { + match y { + 0.0 => Err("Division by zero"), + _ => Ok(x / y), + } + } + """ + And I have some code examples + """ocaml + let rec factorial = function + | 0 -> 1 + | n -> n * factorial (n - 1) + + let () = Printf.printf "%d\n" (factorial 5) + """ + And I have some code examples + """json + { + "calculator": { + "operations": ["add", "subtract", "multiply", "divide"], + "precision": 2, + "enabled": true + } + } + """ + And I have some code examples + """nonexistent-lang + proc divide(x, y) { + return x / y; + } + """ + And I have some code examples + """ + def add(a, b) + a + b + end + """ diff --git a/lib/rouge/lexers/gherkin.rb b/lib/rouge/lexers/gherkin.rb index 35f25bab59..17e2515856 100644 --- a/lib/rouge/lexers/gherkin.rb +++ b/lib/rouge/lexers/gherkin.rb @@ -28,9 +28,24 @@ def self.detect?(text) rule %r/[ \r\t]+/, Text end + state :annotated_string do + rule %r(("""(\S+)\n)(.*?)("""))m do |m| + first_line, lang, content, last_line = m.captures + token Str, first_line + sublexer = Rouge::Lexer.find(lang.strip) + if sublexer + delegate sublexer, content + else + token Str, content + end + token Str, last_line + end + end + state :root do mixin :basic rule %r(\n), Text + mixin :annotated_string rule %r(""".*?""")m, Str rule %r(@[^\s@]+), Name::Tag mixin :has_table diff --git a/spec/lexers/gherkin_spec.rb b/spec/lexers/gherkin_spec.rb index 94ea7cf530..777a0f0dc0 100644 --- a/spec/lexers/gherkin_spec.rb +++ b/spec/lexers/gherkin_spec.rb @@ -44,5 +44,38 @@ assert { tokens[6] == [Token['Name.Variable'], ''] } assert { tokens[7] == [Token['Text'], ')< garbage'] } end + + it 'highlights annotated strings correctly' do + source = <<~GHERKIN + Given I have some code + """ruby + class Calculator + end + """ + GHERKIN + + find_calls = [] + original_find = Rouge::Lexer.method(:find) + Rouge::Lexer.define_singleton_method(:find) do |lang| + find_calls << lang + original_find.call(lang) + end + + tokens = subject.lex(source).to_a + + assert { tokens.size == 11 } + assert { tokens[0][0] == Token['Name.Function'] } + assert { tokens[2][0] == Token['Literal.String'] } + assert { tokens[1][0] == Token['Text'] } + assert { tokens[3][0] == Token['Keyword'] } + assert { tokens[4][0] == Token['Text'] } + assert { tokens[5][0] == Token['Name.Class'] } + assert { tokens[6][0] == Token['Text'] } + assert { tokens[7][0] == Token['Keyword'] } + assert { tokens[8][0] == Token['Text'] } + assert { tokens[9][0] == Token['Literal.String'] } + assert { tokens[10][0] == Token['Text'] } + assert { find_calls.include?('ruby') } + end end end From 9232fbf8bcc616837147fe8a849d17c5796cbc5d Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 18 Mar 2026 01:42:17 -0400 Subject: [PATCH 2/4] move the visual spec into the visual spec file and out of the demo --- lib/rouge/demos/gherkin | 49 ------------------------------------- spec/visual/samples/gherkin | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/lib/rouge/demos/gherkin b/lib/rouge/demos/gherkin index 5e082db8a7..78e4ba6e87 100644 --- a/lib/rouge/demos/gherkin +++ b/lib/rouge/demos/gherkin @@ -15,52 +15,3 @@ Feature: Addition | 20 | 30 | add | 50 | | 2 | 5 | add | 7 | | 0 | 40 | add | 40 | - - Scenario: Code samples - Given I have some code examples - """ruby - class Calculator - def multiply(x, y) - x * y - end - end - """ - And I have some code examples - """rust - fn safe_divide(x: f64, y: f64) -> Result { - match y { - 0.0 => Err("Division by zero"), - _ => Ok(x / y), - } - } - """ - And I have some code examples - """ocaml - let rec factorial = function - | 0 -> 1 - | n -> n * factorial (n - 1) - - let () = Printf.printf "%d\n" (factorial 5) - """ - And I have some code examples - """json - { - "calculator": { - "operations": ["add", "subtract", "multiply", "divide"], - "precision": 2, - "enabled": true - } - } - """ - And I have some code examples - """nonexistent-lang - proc divide(x, y) { - return x / y; - } - """ - And I have some code examples - """ - def add(a, b) - a + b - end - """ diff --git a/spec/visual/samples/gherkin b/spec/visual/samples/gherkin index 759e0b6524..39ff064476 100644 --- a/spec/visual/samples/gherkin +++ b/spec/visual/samples/gherkin @@ -79,3 +79,52 @@ Feature: Highlander Then he (or she) will live forever ;-) Rule: There can be Two (in some cases) + + Scenario: Code samples + Given I have some code examples + """ruby + class Calculator + def multiply(x, y) + x * y + end + end + """ + And I have some code examples + """rust + fn safe_divide(x: f64, y: f64) -> Result { + match y { + 0.0 => Err("Division by zero"), + _ => Ok(x / y), + } + } + """ + And I have some code examples + """ocaml + let rec factorial = function + | 0 -> 1 + | n -> n * factorial (n - 1) + + let () = Printf.printf "%d\n" (factorial 5) + """ + And I have some code examples + """json + { + "calculator": { + "operations": ["add", "subtract", "multiply", "divide"], + "precision": 2, + "enabled": true + } + } + """ + And I have some code examples + """nonexistent-lang + proc divide(x, y) { + return x / y; + } + """ + And I have some code examples + """ + def add(a, b) + a + b + end + """ From 2e0a82e0078fc131136dc71aa587d75b65ef70af Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 18 Mar 2026 01:49:17 -0400 Subject: [PATCH 3/4] style and don't re-scan triple quotes --- lib/rouge/lexers/gherkin.rb | 14 +++++++++----- spec/lexers/gherkin_spec.rb | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/gherkin.rb b/lib/rouge/lexers/gherkin.rb index 17e2515856..567183fd20 100644 --- a/lib/rouge/lexers/gherkin.rb +++ b/lib/rouge/lexers/gherkin.rb @@ -29,16 +29,21 @@ def self.detect?(text) end state :annotated_string do - rule %r(("""(\S+)\n)(.*?)("""))m do |m| - first_line, lang, content, last_line = m.captures - token Str, first_line + rule %r((""")(\S*)(.*?)("""))m do |m| + open, lang, content, close = m.captures + + token Str, open + token Str::Escape, lang + sublexer = Rouge::Lexer.find(lang.strip) + if sublexer delegate sublexer, content else token Str, content end - token Str, last_line + + token Str, close end end @@ -46,7 +51,6 @@ def self.detect?(text) mixin :basic rule %r(\n), Text mixin :annotated_string - rule %r(""".*?""")m, Str rule %r(@[^\s@]+), Name::Tag mixin :has_table mixin :has_examples diff --git a/spec/lexers/gherkin_spec.rb b/spec/lexers/gherkin_spec.rb index 777a0f0dc0..7516553874 100644 --- a/spec/lexers/gherkin_spec.rb +++ b/spec/lexers/gherkin_spec.rb @@ -53,16 +53,16 @@ class Calculator end """ GHERKIN - + find_calls = [] original_find = Rouge::Lexer.method(:find) Rouge::Lexer.define_singleton_method(:find) do |lang| find_calls << lang original_find.call(lang) end - + tokens = subject.lex(source).to_a - + assert { tokens.size == 11 } assert { tokens[0][0] == Token['Name.Function'] } assert { tokens[2][0] == Token['Literal.String'] } From f4339eb2367b3aca491726de0935d537c22b2ad5 Mon Sep 17 00:00:00 2001 From: jneen Date: Wed, 18 Mar 2026 01:56:07 -0400 Subject: [PATCH 4/4] delete a too-brittle spec in favour of the visual spec --- spec/lexers/gherkin_spec.rb | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/spec/lexers/gherkin_spec.rb b/spec/lexers/gherkin_spec.rb index 7516553874..94ea7cf530 100644 --- a/spec/lexers/gherkin_spec.rb +++ b/spec/lexers/gherkin_spec.rb @@ -44,38 +44,5 @@ assert { tokens[6] == [Token['Name.Variable'], ''] } assert { tokens[7] == [Token['Text'], ')< garbage'] } end - - it 'highlights annotated strings correctly' do - source = <<~GHERKIN - Given I have some code - """ruby - class Calculator - end - """ - GHERKIN - - find_calls = [] - original_find = Rouge::Lexer.method(:find) - Rouge::Lexer.define_singleton_method(:find) do |lang| - find_calls << lang - original_find.call(lang) - end - - tokens = subject.lex(source).to_a - - assert { tokens.size == 11 } - assert { tokens[0][0] == Token['Name.Function'] } - assert { tokens[2][0] == Token['Literal.String'] } - assert { tokens[1][0] == Token['Text'] } - assert { tokens[3][0] == Token['Keyword'] } - assert { tokens[4][0] == Token['Text'] } - assert { tokens[5][0] == Token['Name.Class'] } - assert { tokens[6][0] == Token['Text'] } - assert { tokens[7][0] == Token['Keyword'] } - assert { tokens[8][0] == Token['Text'] } - assert { tokens[9][0] == Token['Literal.String'] } - assert { tokens[10][0] == Token['Text'] } - assert { find_calls.include?('ruby') } - end end end