From 01e48173aa5d7ed0cc6c3a8e3c473bf1bb8c5c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Sun, 28 Jan 2024 18:30:22 +0100 Subject: [PATCH 01/17] highlight: Remove duplicated util code --- pkg/highlight/highlighter.go | 72 ++++++------------------------ pkg/highlight/unicode.go | 85 ------------------------------------ 2 files changed, 14 insertions(+), 143 deletions(-) delete mode 100644 pkg/highlight/unicode.go diff --git a/pkg/highlight/highlighter.go b/pkg/highlight/highlighter.go index a13a72610d..024035d340 100644 --- a/pkg/highlight/highlighter.go +++ b/pkg/highlight/highlighter.go @@ -3,53 +3,9 @@ package highlight import ( "regexp" "strings" -) - -func sliceStart(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[totalSize:] - } - - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - - return slc[totalSize:] -} - -func sliceEnd(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[:totalSize] - } - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - - return slc[:totalSize] -} - -// RunePos returns the rune index of a given byte index -// This could cause problems if the byte index is between code points -func runePos(p int, str []byte) int { - if p < 0 { - return 0 - } - if p >= len(str) { - return CharacterCount(str) - } - return CharacterCount(str[:p]) -} + "github.com/zyedidia/micro/v2/internal/util" +) // A State represents the region at the end of a line type State *region @@ -86,7 +42,7 @@ func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { var strbytes []byte if skip != nil { strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { - res := make([]byte, CharacterCount(match)) + res := make([]byte, util.CharacterCount(match)) return res }) } else { @@ -98,20 +54,20 @@ func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { return nil } // return []int{match.Index, match.Index + match.Length} - return []int{runePos(match[0], str), runePos(match[1], str)} + return []int{util.RunePos(str, match[0]), util.RunePos(str, match[1])} } func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { matches := regex.FindAllIndex(str, -1) for i, m := range matches { - matches[i][0] = runePos(m[0], str) - matches[i][1] = runePos(m[1], str) + matches[i][0] = util.RunePos(str, m[0]) + matches[i][1] = util.RunePos(str, m[1]) } return matches } func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) + lineLen := util.CharacterCount(line) if start == 0 { if !statesOnly { if _, ok := highlights[0]; !ok { @@ -146,8 +102,8 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE if !statesOnly { highlights[start+firstLoc[0]] = firstRegion.limitGroup } - h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) + h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), statesOnly) + h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) return highlights } @@ -187,13 +143,13 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE if !statesOnly { highlights[start+loc[1]] = 0 } - h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly) + h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, util.SliceEnd(line, loc[1]), statesOnly) return highlights } if !statesOnly { highlights[start+loc[1]] = curRegion.parent.group } - h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly) + h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, util.SliceEnd(line, loc[1]), curRegion.parent, statesOnly) return highlights } @@ -205,7 +161,7 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE } func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) + lineLen := util.CharacterCount(line) if lineLen == 0 { if canMatchEnd { h.lastRegion = nil @@ -228,8 +184,8 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM if !statesOnly { highlights[start+firstLoc[0]] = firstRegion.limitGroup } - h.highlightEmptyRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) + h.highlightEmptyRegion(highlights, start, false, lineNum, util.SliceStart(line, firstLoc[0]), statesOnly) + h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) return highlights } diff --git a/pkg/highlight/unicode.go b/pkg/highlight/unicode.go deleted file mode 100644 index a18118a6ca..0000000000 --- a/pkg/highlight/unicode.go +++ /dev/null @@ -1,85 +0,0 @@ -package highlight - -import ( - "unicode" - "unicode/utf8" -) - -var minMark = rune(unicode.Mark.R16[0].Lo) - -func isMark(r rune) bool { - // Fast path - if r < minMark { - return false - } - return unicode.In(r, unicode.Mark) -} - -// DecodeCharacter returns the next character from an array of bytes -// A character is a rune along with any accompanying combining runes -func DecodeCharacter(b []byte) (rune, []rune, int) { - r, size := utf8.DecodeRune(b) - b = b[size:] - c, s := utf8.DecodeRune(b) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - b = b[s:] - c, s = utf8.DecodeRune(b) - } - - return r, combc, size -} - -// DecodeCharacterInString returns the next character from a string -// A character is a rune along with any accompanying combining runes -func DecodeCharacterInString(str string) (rune, []rune, int) { - r, size := utf8.DecodeRuneInString(str) - str = str[size:] - c, s := utf8.DecodeRuneInString(str) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - str = str[s:] - c, s = utf8.DecodeRuneInString(str) - } - - return r, combc, size -} - -// CharacterCount returns the number of characters in a byte array -// Similar to utf8.RuneCount but for unicode characters -func CharacterCount(b []byte) int { - s := 0 - - for len(b) > 0 { - r, size := utf8.DecodeRune(b) - if !isMark(r) { - s++ - } - - b = b[size:] - } - - return s -} - -// CharacterCount returns the number of characters in a string -// Similar to utf8.RuneCountInString but for unicode characters -func CharacterCountInString(str string) int { - s := 0 - - for _, r := range str { - if !isMark(r) { - s++ - } - } - - return s -} From 082084edd18d277dc43a88df3dde0b5f2375dc54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Thu, 13 Mar 2025 18:00:30 +0100 Subject: [PATCH 02/17] highlight: Move from pkg to internal source We break away from the assumption that the highlighter will be reused in different projects. --- internal/action/infocomplete.go | 2 +- internal/buffer/buffer.go | 2 +- internal/buffer/line_array.go | 2 +- {pkg => internal}/highlight/highlighter.go | 0 {pkg => internal}/highlight/parser.go | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename {pkg => internal}/highlight/highlighter.go (100%) rename {pkg => internal}/highlight/parser.go (100%) diff --git a/internal/action/infocomplete.go b/internal/action/infocomplete.go index b8ee879a30..783132fc9d 100644 --- a/internal/action/infocomplete.go +++ b/internal/action/infocomplete.go @@ -7,8 +7,8 @@ import ( "github.com/zyedidia/micro/v2/internal/buffer" "github.com/zyedidia/micro/v2/internal/config" + "github.com/zyedidia/micro/v2/internal/highlight" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" ) // This file is meant (for now) for autocompletion in command mode, not diff --git a/internal/buffer/buffer.go b/internal/buffer/buffer.go index 4226d972cd..39b14c944b 100644 --- a/internal/buffer/buffer.go +++ b/internal/buffer/buffer.go @@ -21,10 +21,10 @@ import ( dmp "github.com/sergi/go-diff/diffmatchpatch" "github.com/zyedidia/micro/v2/internal/config" + "github.com/zyedidia/micro/v2/internal/highlight" ulua "github.com/zyedidia/micro/v2/internal/lua" "github.com/zyedidia/micro/v2/internal/screen" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" "golang.org/x/text/encoding" "golang.org/x/text/encoding/htmlindex" "golang.org/x/text/encoding/unicode" diff --git a/internal/buffer/line_array.go b/internal/buffer/line_array.go index b65213b805..97ced52122 100644 --- a/internal/buffer/line_array.go +++ b/internal/buffer/line_array.go @@ -6,8 +6,8 @@ import ( "io" "sync" + "github.com/zyedidia/micro/v2/internal/highlight" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" ) // Finds the byte index of the nth rune in a byte slice diff --git a/pkg/highlight/highlighter.go b/internal/highlight/highlighter.go similarity index 100% rename from pkg/highlight/highlighter.go rename to internal/highlight/highlighter.go diff --git a/pkg/highlight/parser.go b/internal/highlight/parser.go similarity index 100% rename from pkg/highlight/parser.go rename to internal/highlight/parser.go From 5ae749491d5cc120847ff84010d2c9316787dba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:08:23 +0100 Subject: [PATCH 03/17] highlighter: Perform sliceEnd() in nested region only once --- internal/highlight/highlighter.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 024035d340..df27a2ec42 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -102,8 +102,9 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE if !statesOnly { highlights[start+firstLoc[0]] = firstRegion.limitGroup } - h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) + slice := util.SliceEnd(line, firstLoc[1]) + h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, slice, statesOnly) + h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, slice, firstRegion, statesOnly) return highlights } From 3dd52a47d346f793f7ddd44f21f7c8b7fffdce3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:48:46 +0100 Subject: [PATCH 04/17] highlighter: Rename firstLoc/Region into nestedLoc/Region for readability --- internal/highlight/highlighter.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index df27a2ec42..681d24fac5 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -76,35 +76,35 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE } } - var firstRegion *region - firstLoc := []int{lineLen, 0} + var nestedRegion *region + nestedLoc := []int{lineLen, 0} searchNesting := true endLoc := findIndex(curRegion.end, curRegion.skip, line) if endLoc != nil { if start == endLoc[0] { searchNesting = false } else { - firstLoc = endLoc + nestedLoc = endLoc } } if searchNesting { for _, r := range curRegion.rules.regions { loc := findIndex(r.start, r.skip, line) if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r + if loc[0] < nestedLoc[0] { + nestedLoc = loc + nestedRegion = r } } } } - if firstRegion != nil && firstLoc[0] != lineLen { + if nestedRegion != nil && nestedLoc[0] != lineLen { if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup + highlights[start+nestedLoc[0]] = nestedRegion.limitGroup } - slice := util.SliceEnd(line, firstLoc[1]) - h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, slice, statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, slice, firstRegion, statesOnly) + slice := util.SliceEnd(line, nestedLoc[1]) + h.highlightEmptyRegion(highlights, start+nestedLoc[1], canMatchEnd, lineNum, slice, statesOnly) + h.highlightRegion(highlights, start+nestedLoc[1], canMatchEnd, lineNum, slice, nestedRegion, statesOnly) return highlights } From 4e8eed81c47fd2ab9a4a8b7791d45f30b56c6709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:29:15 +0100 Subject: [PATCH 05/17] highlighter: Remove canMatchEnd since it's not really useful --- internal/highlight/highlighter.go | 52 ++++++++++++------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 681d24fac5..51648a037c 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -66,7 +66,7 @@ func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { return matches } -func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { +func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { lineLen := util.CharacterCount(line) if start == 0 { if !statesOnly { @@ -103,8 +103,8 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE highlights[start+nestedLoc[0]] = nestedRegion.limitGroup } slice := util.SliceEnd(line, nestedLoc[1]) - h.highlightEmptyRegion(highlights, start+nestedLoc[1], canMatchEnd, lineNum, slice, statesOnly) - h.highlightRegion(highlights, start+nestedLoc[1], canMatchEnd, lineNum, slice, nestedRegion, statesOnly) + h.highlightEmptyRegion(highlights, start+nestedLoc[1], lineNum, slice, statesOnly) + h.highlightRegion(highlights, start+nestedLoc[1], lineNum, slice, nestedRegion, statesOnly) return highlights } @@ -144,29 +144,25 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE if !statesOnly { highlights[start+loc[1]] = 0 } - h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, util.SliceEnd(line, loc[1]), statesOnly) + h.highlightEmptyRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), statesOnly) return highlights } if !statesOnly { highlights[start+loc[1]] = curRegion.parent.group } - h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, util.SliceEnd(line, loc[1]), curRegion.parent, statesOnly) + h.highlightRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), curRegion.parent, statesOnly) return highlights } - if canMatchEnd { - h.lastRegion = curRegion - } + h.lastRegion = curRegion return highlights } -func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch { +func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, lineNum int, line []byte, statesOnly bool) LineMatch { lineLen := util.CharacterCount(line) if lineLen == 0 { - if canMatchEnd { - h.lastRegion = nil - } + h.lastRegion = nil return highlights } @@ -185,16 +181,12 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM if !statesOnly { highlights[start+firstLoc[0]] = firstRegion.limitGroup } - h.highlightEmptyRegion(highlights, start, false, lineNum, util.SliceStart(line, firstLoc[0]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) + h.highlightEmptyRegion(highlights, start, lineNum, util.SliceStart(line, firstLoc[0]), statesOnly) + h.highlightRegion(highlights, start+firstLoc[1], lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) return highlights } if statesOnly { - if canMatchEnd { - h.lastRegion = nil - } - return highlights } @@ -215,10 +207,6 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM } } - if canMatchEnd { - h.lastRegion = nil - } - return highlights } @@ -235,9 +223,9 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { highlights := make(LineMatch) if i == 0 || h.lastRegion == nil { - lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, true, i, line, false)) + lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, i, line, false)) } else { - lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, true, i, line, h.lastRegion, false)) + lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, i, line, h.lastRegion, false)) } } @@ -257,9 +245,9 @@ func (h *Highlighter) HighlightStates(input LineStates) { // highlights := make(LineMatch) if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) + h.highlightEmptyRegion(nil, 0, i, line, true) } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) + h.highlightRegion(nil, 0, i, line, h.lastRegion, true) } curState := h.lastRegion @@ -285,9 +273,9 @@ func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int) var match LineMatch if i == 0 || input.State(i-1) == nil { - match = h.highlightEmptyRegion(highlights, 0, true, i, line, false) + match = h.highlightEmptyRegion(highlights, 0, i, line, false) } else { - match = h.highlightRegion(highlights, 0, true, i, line, input.State(i-1), false) + match = h.highlightRegion(highlights, 0, i, line, input.State(i-1), false) } input.SetMatch(i, match) @@ -321,9 +309,9 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { // var match LineMatch if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) + h.highlightEmptyRegion(nil, 0, i, line, true) } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) + h.highlightRegion(nil, 0, i, line, h.lastRegion, true) } curState := h.lastRegion lastState := input.State(i) @@ -354,9 +342,9 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { var match LineMatch if lineN == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, true, lineN, line, false) + match = h.highlightEmptyRegion(highlights, 0, lineN, line, false) } else { - match = h.highlightRegion(highlights, 0, true, lineN, line, h.lastRegion, false) + match = h.highlightRegion(highlights, 0, lineN, line, h.lastRegion, false) } curState := h.lastRegion From cee8c0d26587b4801874891420d6b9baf0f8b3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 17 Jan 2024 23:01:16 +0100 Subject: [PATCH 06/17] highlighter: Combine HighlightStates and HighlightMatches to Highlight The most expensive path in the highlighting process is the regex parsing step which was invoked for every line twice due to the split approach. Since this can be done in one step the highlighting effort can be reduced by around 50 percent! --- internal/buffer/buffer.go | 9 +--- internal/highlight/highlighter.go | 83 ++++--------------------------- 2 files changed, 13 insertions(+), 79 deletions(-) diff --git a/internal/buffer/buffer.go b/internal/buffer/buffer.go index 39b14c944b..ba9b4a6b20 100644 --- a/internal/buffer/buffer.go +++ b/internal/buffer/buffer.go @@ -150,11 +150,7 @@ func (b *SharedBuffer) MarkModified(start, end int) { end = util.Clamp(end, 0, len(b.lines)-1) if b.Settings["syntax"].(bool) && b.SyntaxDef != nil { - l := -1 - for i := start; i <= end; i++ { - l = util.Max(b.Highlighter.ReHighlightStates(b, i), l) - } - b.Highlighter.HighlightMatches(b, start, l) + b.Highlighter.Highlight(b, start, end) } for i := start; i <= end; i++ { @@ -960,8 +956,7 @@ func (b *Buffer) UpdateRules() { b.Highlighter = highlight.NewHighlighter(b.SyntaxDef) if b.Settings["syntax"].(bool) { go func() { - b.Highlighter.HighlightStates(b) - b.Highlighter.HighlightMatches(b, 0, b.End().Y) + b.Highlighter.Highlight(b, 0, b.End().Y) screen.Redraw() }() } diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 51648a037c..f3c45aa79b 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -232,63 +232,9 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { return lineMatches } -// HighlightStates correctly sets all states for the buffer -func (h *Highlighter) HighlightStates(input LineStates) { - for i := 0; ; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - // highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, i, line, true) - } else { - h.highlightRegion(nil, 0, i, line, h.lastRegion, true) - } - - curState := h.lastRegion - - input.SetState(i, curState) - input.Unlock() - } -} - -// HighlightMatches sets the matches for each line from startline to endline +// Highlight sets the state and matches for each line from startline to endline // It sets all other matches in the buffer to nil to conserve memory -// This assumes that all the states are set correctly -func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int) { - for i := startline; i <= endline; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - highlights := make(LineMatch) - - var match LineMatch - if i == 0 || input.State(i-1) == nil { - match = h.highlightEmptyRegion(highlights, 0, i, line, false) - } else { - match = h.highlightRegion(highlights, 0, i, line, input.State(i-1), false) - } - - input.SetMatch(i, match) - input.Unlock() - } -} - -// ReHighlightStates will scan down from `startline` and set the appropriate end of line state -// for each line until it comes across a line whose state does not change -// returns the number of the final line -func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { - // lines := input.LineData() - +func (h *Highlighter) Highlight(input LineStates, startline, endline int) { h.lastRegion = nil if startline > 0 { input.Lock() @@ -297,7 +243,8 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { } input.Unlock() } - for i := startline; ; i++ { + + for i := startline; i <= endline; i++ { input.Lock() if i >= input.LinesNum() { input.Unlock() @@ -305,26 +252,19 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { } line := input.LineBytes(i) - // highlights := make(LineMatch) + highlights := make(LineMatch) - // var match LineMatch + var match LineMatch if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, i, line, true) + match = h.highlightEmptyRegion(highlights, 0, i, line, false) } else { - h.highlightRegion(nil, 0, i, line, h.lastRegion, true) + match = h.highlightRegion(highlights, 0, i, line, h.lastRegion, false) } - curState := h.lastRegion - lastState := input.State(i) - input.SetState(i, curState) + input.SetState(i, h.lastRegion) + input.SetMatch(i, match) input.Unlock() - - if curState == lastState { - return i - } } - - return input.LinesNum() - 1 } // ReHighlightLine will rehighlight the state and match for a single line @@ -346,8 +286,7 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { } else { match = h.highlightRegion(highlights, 0, lineN, line, h.lastRegion, false) } - curState := h.lastRegion + input.SetState(lineN, h.lastRegion) input.SetMatch(lineN, match) - input.SetState(lineN, curState) } From 134fce5277147a3109e1bcdcfe927271be3587e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 17 Jan 2024 23:14:21 +0100 Subject: [PATCH 07/17] highlighter: Remove statesOnly since it's not needed any longer --- internal/highlight/highlighter.go | 92 +++++++++++++------------------ 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index f3c45aa79b..ebbfa023ac 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -66,13 +66,11 @@ func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { return matches } -func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { +func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) LineMatch { lineLen := util.CharacterCount(line) if start == 0 { - if !statesOnly { - if _, ok := highlights[0]; !ok { - highlights[0] = curRegion.group - } + if _, ok := highlights[0]; !ok { + highlights[0] = curRegion.group } } @@ -99,58 +97,48 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum i } } if nestedRegion != nil && nestedLoc[0] != lineLen { - if !statesOnly { - highlights[start+nestedLoc[0]] = nestedRegion.limitGroup - } + highlights[start+nestedLoc[0]] = nestedRegion.limitGroup slice := util.SliceEnd(line, nestedLoc[1]) - h.highlightEmptyRegion(highlights, start+nestedLoc[1], lineNum, slice, statesOnly) - h.highlightRegion(highlights, start+nestedLoc[1], lineNum, slice, nestedRegion, statesOnly) + h.highlightEmptyRegion(highlights, start+nestedLoc[1], lineNum, slice) + h.highlightRegion(highlights, start+nestedLoc[1], lineNum, slice, nestedRegion) return highlights } - if !statesOnly { - fullHighlights := make([]Group, lineLen) - for i := 0; i < len(fullHighlights); i++ { - fullHighlights[i] = curRegion.group - } + fullHighlights := make([]Group, lineLen) + for i := 0; i < len(fullHighlights); i++ { + fullHighlights[i] = curRegion.group + } - if searchNesting { - for _, p := range curRegion.rules.patterns { - if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - if (endLoc == nil) || (m[0] < endLoc[0]) { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } + if searchNesting { + for _, p := range curRegion.rules.patterns { + if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup { + matches := findAllIndex(p.regex, line) + for _, m := range matches { + if (endLoc == nil) || (m[0] < endLoc[0]) { + for i := m[0]; i < m[1]; i++ { + fullHighlights[i] = p.group } } } } } - for i, h := range fullHighlights { - if i == 0 || h != fullHighlights[i-1] { - highlights[start+i] = h - } + } + for i, h := range fullHighlights { + if i == 0 || h != fullHighlights[i-1] { + highlights[start+i] = h } } loc := endLoc if loc != nil { - if !statesOnly { - highlights[start+loc[0]] = curRegion.limitGroup - } + highlights[start+loc[0]] = curRegion.limitGroup if curRegion.parent == nil { - if !statesOnly { - highlights[start+loc[1]] = 0 - } - h.highlightEmptyRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), statesOnly) + highlights[start+loc[1]] = 0 + h.highlightEmptyRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1])) return highlights } - if !statesOnly { - highlights[start+loc[1]] = curRegion.parent.group - } - h.highlightRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), curRegion.parent, statesOnly) + highlights[start+loc[1]] = curRegion.parent.group + h.highlightRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), curRegion.parent) return highlights } @@ -159,7 +147,7 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum i return highlights } -func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, lineNum int, line []byte, statesOnly bool) LineMatch { +func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, lineNum int, line []byte) LineMatch { lineLen := util.CharacterCount(line) if lineLen == 0 { h.lastRegion = nil @@ -178,15 +166,9 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, line } } if firstRegion != nil && firstLoc[0] != lineLen { - if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup - } - h.highlightEmptyRegion(highlights, start, lineNum, util.SliceStart(line, firstLoc[0]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion, statesOnly) - return highlights - } - - if statesOnly { + highlights[start+firstLoc[0]] = firstRegion.limitGroup + h.highlightEmptyRegion(highlights, start, lineNum, util.SliceStart(line, firstLoc[0])) + h.highlightRegion(highlights, start+firstLoc[1], lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion) return highlights } @@ -223,9 +205,9 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { highlights := make(LineMatch) if i == 0 || h.lastRegion == nil { - lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, i, line, false)) + lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, i, line)) } else { - lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, i, line, h.lastRegion, false)) + lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, i, line, h.lastRegion)) } } @@ -256,9 +238,9 @@ func (h *Highlighter) Highlight(input LineStates, startline, endline int) { var match LineMatch if i == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, i, line, false) + match = h.highlightEmptyRegion(highlights, 0, i, line) } else { - match = h.highlightRegion(highlights, 0, i, line, h.lastRegion, false) + match = h.highlightRegion(highlights, 0, i, line, h.lastRegion) } input.SetState(i, h.lastRegion) @@ -282,9 +264,9 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { var match LineMatch if lineN == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, lineN, line, false) + match = h.highlightEmptyRegion(highlights, 0, lineN, line) } else { - match = h.highlightRegion(highlights, 0, lineN, line, h.lastRegion, false) + match = h.highlightRegion(highlights, 0, lineN, line, h.lastRegion) } input.SetState(lineN, h.lastRegion) From 29b48cd706753a92bb14eaa764bd6fe11f79c593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Fri, 26 Jan 2024 00:02:54 +0100 Subject: [PATCH 08/17] highlighter: Rework of matching approach to speed up the processing The old approach was to slide over the line by (empty) region over region one after another. Step down into a possible available nested part and do the same. The new approach iterates over all available region definitions and shall find all occurrences of the same region at the same time (`findAllIndex()`) and not just the first (`findIndex()`). This will allow marking a region already in the moment it isn't the first region at the line. After an region has been identified all nested parts are considered the same way. Simple example (no nesting): 1. region start=" end=" 2. region start=' end=' 3. region start=< end=> The "first", "second" and "third" are highlighted in one round. This 'highlights after ""' while and "highlights first". --- internal/highlight/highlighter.go | 275 ++++++++++++++++++------------ internal/util/util.go | 10 ++ 2 files changed, 175 insertions(+), 110 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index ebbfa023ac..55d1fc33a8 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -24,6 +24,8 @@ type LineStates interface { // A Highlighter contains the information needed to highlight a string type Highlighter struct { lastRegion *region + lastStart int + lastEnd int Def *Def } @@ -57,8 +59,18 @@ func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { return []int{util.RunePos(str, match[0]), util.RunePos(str, match[1])} } -func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { - matches := regex.FindAllIndex(str, -1) +func findAllIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) [][]int { + var strbytes []byte + if skip != nil { + strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { + res := make([]byte, util.CharacterCount(match)) + return res + }) + } else { + strbytes = str + } + + matches := regex.FindAllIndex(strbytes, -1) for i, m := range matches { matches[i][0] = util.RunePos(str, m[0]) matches[i][1] = util.RunePos(str, m[1]) @@ -66,126 +78,183 @@ func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { return matches } -func (h *Highlighter) highlightRegion(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) LineMatch { - lineLen := util.CharacterCount(line) - if start == 0 { - if _, ok := highlights[0]; !ok { - highlights[0] = curRegion.group +func (h *Highlighter) highlightRange(fullHighlights []Group, start int, end int, group Group) { + if start <= end && end <= len(fullHighlights) { + for i := start; i < end; i++ { + fullHighlights[i] = group } } +} - var nestedRegion *region - nestedLoc := []int{lineLen, 0} - searchNesting := true - endLoc := findIndex(curRegion.end, curRegion.skip, line) - if endLoc != nil { - if start == endLoc[0] { - searchNesting = false - } else { - nestedLoc = endLoc - } +func (h *Highlighter) highlightPatterns(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region) { + lineLen := util.CharacterCount(line) + if lineLen == 0 { + return } - if searchNesting { - for _, r := range curRegion.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < nestedLoc[0] { - nestedLoc = loc - nestedRegion = r - } - } + + var patterns []*pattern + if curRegion == nil { + patterns = h.Def.rules.patterns + } else { + patterns = curRegion.rules.patterns + } + + for _, p := range patterns { + matches := findAllIndex(p.regex, nil, line) + for _, m := range matches { + h.highlightRange(fullHighlights, start+m[0], start+m[1], p.group) } } - if nestedRegion != nil && nestedLoc[0] != lineLen { - highlights[start+nestedLoc[0]] = nestedRegion.limitGroup - slice := util.SliceEnd(line, nestedLoc[1]) - h.highlightEmptyRegion(highlights, start+nestedLoc[1], lineNum, slice) - h.highlightRegion(highlights, start+nestedLoc[1], lineNum, slice, nestedRegion) - return highlights +} + +func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { + lineLen := util.CharacterCount(line) + if lineLen == 0 { + return } - fullHighlights := make([]Group, lineLen) - for i := 0; i < len(fullHighlights); i++ { - fullHighlights[i] = curRegion.group + if nestedRegion { + h.highlightPatterns(fullHighlights, start, lineNum, line, curRegion) + } else { + h.highlightPatterns(fullHighlights, start, lineNum, line, nil) } - if searchNesting { - for _, p := range curRegion.rules.patterns { - if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - if (endLoc == nil) || (m[0] < endLoc[0]) { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group + for _, r := range regions { + if !nestedRegion && curRegion != nil && curRegion != r { + continue + } + startMatches := findAllIndex(r.start, r.skip, line) + endMatches := findAllIndex(r.end, r.skip, line) + samePattern := false + startLoop: + for startIdx := 0; startIdx < len(startMatches); startIdx++ { + startMatch := startMatches[startIdx] + for endIdx := 0; endIdx < len(endMatches); endIdx++ { + endMatch := endMatches[endIdx] + if startMatch[0] == endMatch[0] { + // start and end are the same (pattern) + samePattern = true + if len(startMatches) == len(endMatches) { + // special case in the moment both are the same + if curRegion == r { + if len(startMatches) > 1 { + // end < start + continue startLoop + } else if len(startMatches) > 0 { + // ... end + startIdx = len(startMatches) + continue startLoop + } + } else { + // start ... or start < end } } + } else if startMatch[1] <= endMatch[0] { + if !nestedRegion && h.lastStart < start+startMatch[0] && start+startMatch[0] < h.lastEnd { + continue + } + // start and end at the current line + update := false + if h.lastStart == 0 || h.lastStart < start+endMatch[1] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + endMatch[1] + update = true + } + h.highlightRange(fullHighlights, start+startMatch[0], start+endMatch[1], r.limitGroup) + h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) + if samePattern { + startIdx += 1 + } + if update { + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + continue startLoop + } else if endMatch[1] <= startMatch[0] { + if start+endMatch[0] < h.lastEnd || curRegion == nil { + continue + } + // start and end at the current line, but switched + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) + h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + h.highlightPatterns(fullHighlights, start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + } + if nestedRegion || start+startMatch[0] <= h.lastStart || h.lastEnd <= start+startMatch[0] { + // start at the current line + h.lastStart = start + startMatch[0] + h.lastEnd = start + lineLen + h.highlightRange(fullHighlights, start+startMatch[0], start+lineLen, r.limitGroup) + h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) + if h.lastStart == 0 || h.lastStart <= start+startMatch[0] { + h.lastRegion = r } } } - } - for i, h := range fullHighlights { - if i == 0 || h != fullHighlights[i-1] { - highlights[start+i] = h + if curRegion == r { + if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && (len(startMatches) == len(endMatches))) { + for _, endMatch := range endMatches { + // end at the current line + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) + h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + h.highlightRegions(fullHighlights, start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) + break + } + } else if len(startMatches) == 0 && len(endMatches) == 0 { + // no start and end found in this region + h.highlightRange(fullHighlights, start, start+lineLen, curRegion.group) + } } } - loc := endLoc - if loc != nil { - highlights[start+loc[0]] = curRegion.limitGroup - if curRegion.parent == nil { - highlights[start+loc[1]] = 0 - h.highlightEmptyRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1])) - return highlights + if curRegion != nil && !nestedRegion { + // current region still open + if curRegion.rules != nil { + h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, curRegion.rules.regions, true) + } + if curRegion == h.lastRegion && curRegion.parent != nil { + var regions []*region + regions = append(regions, curRegion) + h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, regions, true) } - highlights[start+loc[1]] = curRegion.parent.group - h.highlightRegion(highlights, start+loc[1], lineNum, util.SliceEnd(line, loc[1]), curRegion.parent) - return highlights } - - h.lastRegion = curRegion - - return highlights } -func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, lineNum int, line []byte) LineMatch { +func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) LineMatch { lineLen := util.CharacterCount(line) if lineLen == 0 { - h.lastRegion = nil return highlights } - var firstRegion *region - firstLoc := []int{lineLen, 0} - for _, r := range h.Def.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r - } - } - } - if firstRegion != nil && firstLoc[0] != lineLen { - highlights[start+firstLoc[0]] = firstRegion.limitGroup - h.highlightEmptyRegion(highlights, start, lineNum, util.SliceStart(line, firstLoc[0])) - h.highlightRegion(highlights, start+firstLoc[1], lineNum, util.SliceEnd(line, firstLoc[1]), firstRegion) - return highlights - } + h.lastStart = 0 + h.lastEnd = 0 + + fullHighlights := make([]Group, lineLen) + h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, h.Def.rules.regions, false) - fullHighlights := make([]Group, len(line)) - for _, p := range h.Def.rules.patterns { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } - } - } for i, h := range fullHighlights { if i == 0 || h != fullHighlights[i-1] { - // if _, ok := highlights[start+i]; !ok { - highlights[start+i] = h - // } + highlights[i] = h } } @@ -197,18 +266,14 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, line // more advanced syntax highlighting. They are optimized for quick rehighlighting of the same // text with minor changes made func (h *Highlighter) HighlightString(input string) []LineMatch { + h.lastRegion = nil lines := strings.Split(input, "\n") var lineMatches []LineMatch for i := 0; i < len(lines); i++ { line := []byte(lines[i]) highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, i, line)) - } else { - lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, i, line, h.lastRegion)) - } + lineMatches = append(lineMatches, h.highlight(highlights, 0, i, line, h.lastRegion)) } return lineMatches @@ -236,12 +301,7 @@ func (h *Highlighter) Highlight(input LineStates, startline, endline int) { line := input.LineBytes(i) highlights := make(LineMatch) - var match LineMatch - if i == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, i, line) - } else { - match = h.highlightRegion(highlights, 0, i, line, h.lastRegion) - } + match := h.highlight(highlights, 0, i, line, h.lastRegion) input.SetState(i, h.lastRegion) input.SetMatch(i, match) @@ -262,12 +322,7 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { h.lastRegion = input.State(lineN - 1) } - var match LineMatch - if lineN == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, lineN, line) - } else { - match = h.highlightRegion(highlights, 0, lineN, line, h.lastRegion) - } + match := h.highlight(highlights, 0, lineN, line, h.lastRegion) input.SetState(lineN, h.lastRegion) input.SetMatch(lineN, match) diff --git a/internal/util/util.go b/internal/util/util.go index f2cb2a99d9..0405e37d24 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -171,6 +171,16 @@ func SliceStartStr(str string, index int) string { return str[:totalSize] } +// SliceStartEnd combines SliceStart and SliceEnd into one +func SliceStartEnd(slc []byte, start int, end int) []byte { + return SliceEnd(SliceStart(slc, end), start) +} + +// SliceStartEndStr is the same as SliceStartEnd but for strings +func SliceStartEndStr(str string, start int, end int) string { + return SliceEndStr(SliceStartStr(str, end), start) +} + // SliceVisualEnd will take a byte slice and slice off the start // up to a given visual index. If the index is in the middle of a // rune the number of visual columns into the rune will be returned From ddff579df30f9dff660d85ca830357d32b77b012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:12:07 +0100 Subject: [PATCH 09/17] highlighter: Add disabled debug logs for faster analysis --- internal/highlight/highlighter.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 55d1fc33a8..5893c2263a 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -1,6 +1,7 @@ package highlight import ( + // "log" "regexp" "strings" @@ -82,12 +83,14 @@ func (h *Highlighter) highlightRange(fullHighlights []Group, start int, end int, if start <= end && end <= len(fullHighlights) { for i := start; i < end; i++ { fullHighlights[i] = group + // log.Println("fullHighlights[", i, "]:", group) } } } func (h *Highlighter) highlightPatterns(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region) { lineLen := util.CharacterCount(line) + // log.Println("highlightPatterns: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { return } @@ -109,6 +112,7 @@ func (h *Highlighter) highlightPatterns(fullHighlights []Group, start int, lineN func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { lineLen := util.CharacterCount(line) + // log.Println("highlightRegions: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { return } @@ -120,6 +124,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu } for _, r := range regions { + // log.Println("r.start:", r.start.String(), "r.end:", r.end.String()) if !nestedRegion && curRegion != nil && curRegion != r { continue } @@ -128,11 +133,14 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu samePattern := false startLoop: for startIdx := 0; startIdx < len(startMatches); startIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches)) startMatch := startMatches[startIdx] for endIdx := 0; endIdx < len(endMatches); endIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches), "/ endIdx:", endIdx, "of", len(endMatches), "/ h.lastStart:", h.lastStart, "/ h.lastEnd:", h.lastEnd) endMatch := endMatches[endIdx] if startMatch[0] == endMatch[0] { // start and end are the same (pattern) + // log.Println("start == end") samePattern = true if len(startMatches) == len(endMatches) { // special case in the moment both are the same @@ -154,6 +162,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu continue } // start and end at the current line + // log.Println("start < end") update := false if h.lastStart == 0 || h.lastStart < start+endMatch[1] { h.lastStart = start + startMatch[0] @@ -179,6 +188,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu continue } // start and end at the current line, but switched + // log.Println("end < start") h.lastStart = start h.lastEnd = start + endMatch[1] h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) @@ -194,6 +204,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu } if nestedRegion || start+startMatch[0] <= h.lastStart || h.lastEnd <= start+startMatch[0] { // start at the current line + // log.Println("start ...") h.lastStart = start + startMatch[0] h.lastEnd = start + lineLen h.highlightRange(fullHighlights, start+startMatch[0], start+lineLen, r.limitGroup) @@ -207,6 +218,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && (len(startMatches) == len(endMatches))) { for _, endMatch := range endMatches { // end at the current line + // log.Println("... end") h.lastStart = start h.lastEnd = start + endMatch[1] h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) @@ -229,6 +241,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu if curRegion != nil && !nestedRegion { // current region still open + // log.Println("...") if curRegion.rules != nil { h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, curRegion.rules.regions, true) } @@ -242,6 +255,7 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) LineMatch { lineLen := util.CharacterCount(line) + // log.Println("highlight: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { return highlights } From f2286d078bcf7d5c36948b41b433538d8e680e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:16:10 +0100 Subject: [PATCH 10/17] highlighter: Fix `limitGroup` behavior of regions This is just my interpretation of `limitGroup` as it might be intended a long time ago, but this interpretation could be wrong, since it is not really documented how it should be. --- internal/highlight/highlighter.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 5893c2263a..eae5be898d 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -169,7 +169,9 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu h.lastEnd = start + endMatch[1] update = true } - h.highlightRange(fullHighlights, start+startMatch[0], start+endMatch[1], r.limitGroup) + h.highlightRange(fullHighlights, start+startMatch[0], start+startMatch[1], r.limitGroup) + h.highlightRange(fullHighlights, start+startMatch[1], start+endMatch[0], r.group) + h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) if samePattern { startIdx += 1 @@ -191,7 +193,8 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // log.Println("end < start") h.lastStart = start h.lastEnd = start + endMatch[1] - h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) + h.highlightRange(fullHighlights, start, start+endMatch[0], r.group) + h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) h.highlightPatterns(fullHighlights, start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) if curRegion != nil { @@ -207,7 +210,8 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // log.Println("start ...") h.lastStart = start + startMatch[0] h.lastEnd = start + lineLen - h.highlightRange(fullHighlights, start+startMatch[0], start+lineLen, r.limitGroup) + h.highlightRange(fullHighlights, start+startMatch[0], start+startMatch[1], r.limitGroup) + h.highlightRange(fullHighlights, start+startMatch[1], start+lineLen, r.group) h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) if h.lastStart == 0 || h.lastStart <= start+startMatch[0] { h.lastRegion = r @@ -221,7 +225,8 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // log.Println("... end") h.lastStart = start h.lastEnd = start + endMatch[1] - h.highlightRange(fullHighlights, start, start+endMatch[1], r.limitGroup) + h.highlightRange(fullHighlights, start, start+endMatch[0], r.group) + h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) if curRegion != nil { h.lastRegion = r.parent From 1980ff03d16a86a47e9b3edf1476553988a60467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:53:06 +0100 Subject: [PATCH 11/17] highlighter: Add capability to remove already captured groups This is necessary in the moment sibling regions overlap each other. --- internal/highlight/highlighter.go | 140 ++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 36 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index eae5be898d..87d13a10f1 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -22,12 +22,21 @@ type LineStates interface { Unlock() } +// highlightStorage is used to store the found ranges +type highlightStorage struct { + start int + end int + group Group + region *region +} + // A Highlighter contains the information needed to highlight a string type Highlighter struct { lastRegion *region lastStart int lastEnd int Def *Def + storage []highlightStorage } // NewHighlighter returns a new highlighter from the given syntax definition @@ -79,16 +88,64 @@ func findAllIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) [][]int return matches } -func (h *Highlighter) highlightRange(fullHighlights []Group, start int, end int, group Group) { - if start <= end && end <= len(fullHighlights) { - for i := start; i < end; i++ { - fullHighlights[i] = group - // log.Println("fullHighlights[", i, "]:", group) +func (h *Highlighter) removeRange(start int, end int, removeStart int) { + removeEnd := removeStart + for i := removeStart; i < len(h.storage); i++ { + next := h.storage[i] + if start < next.start && next.start < end { + // log.Println("removed: start:", next.start, "end:", next.end, "group:", next.group) + removeEnd++ } } + if removeStart < removeEnd { + h.storage = append(h.storage[:removeStart], h.storage[removeEnd:]...) + } } -func (h *Highlighter) highlightPatterns(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region) { +func (h *Highlighter) storeRange(start int, end int, group Group, region *region) { + // log.Println("storeRange: start:", start, "end:", end, "group:", group) + for k, e := range h.storage { + if e.region != nil && region != nil { + if e.region.parent == region.parent { + if region == e.region { + if group == e.group && start == e.end { + // same region, update ... + h.storage[k].end = end + // log.Println("exchanged to: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + return + } + } else { + // sibling regions, search for overlaps ... + if start < e.start && end > e.start { + // overlap + } else if (start < e.start && end >= e.end) || (start <= e.start && end > e.end) { + // larger match + } else if start >= e.start && end <= e.end { + // smaller or same match + return + } else { + continue + } + + // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) + h.storage[k] = highlightStorage{start, end, group, region} + + // check and remove follow-ups matching the same + h.removeRange(start, end, k + 1) + return + } + } else { + if region.parent != e.region && start >= e.start && end <= e.end { + return + } + } + } + } + + h.storage = append(h.storage, highlightStorage{start, end, group, region}) +} + +func (h *Highlighter) highlightPatterns(start int, lineNum int, line []byte, curRegion *region) { lineLen := util.CharacterCount(line) // log.Println("highlightPatterns: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { @@ -105,12 +162,12 @@ func (h *Highlighter) highlightPatterns(fullHighlights []Group, start int, lineN for _, p := range patterns { matches := findAllIndex(p.regex, nil, line) for _, m := range matches { - h.highlightRange(fullHighlights, start+m[0], start+m[1], p.group) + h.storeRange(start+m[0], start+m[1], p.group, nil) } } } -func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { +func (h *Highlighter) highlightRegions(start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { lineLen := util.CharacterCount(line) // log.Println("highlightRegions: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { @@ -118,11 +175,12 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu } if nestedRegion { - h.highlightPatterns(fullHighlights, start, lineNum, line, curRegion) + h.highlightPatterns(start, lineNum, line, curRegion) } else { - h.highlightPatterns(fullHighlights, start, lineNum, line, nil) + h.highlightPatterns(start, lineNum, line, nil) } +regionLoop: for _, r := range regions { // log.Println("r.start:", r.start.String(), "r.end:", r.end.String()) if !nestedRegion && curRegion != nil && curRegion != r { @@ -164,15 +222,15 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // start and end at the current line // log.Println("start < end") update := false - if h.lastStart == 0 || h.lastStart < start+endMatch[1] { + if h.lastStart == -1 || h.lastStart < start+endMatch[1] { h.lastStart = start + startMatch[0] h.lastEnd = start + endMatch[1] update = true } - h.highlightRange(fullHighlights, start+startMatch[0], start+startMatch[1], r.limitGroup) - h.highlightRange(fullHighlights, start+startMatch[1], start+endMatch[0], r.group) - h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) - h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r) + h.storeRange(start+startMatch[1], start+endMatch[0], r.group, r) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) if samePattern { startIdx += 1 } @@ -193,10 +251,10 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // log.Println("end < start") h.lastStart = start h.lastEnd = start + endMatch[1] - h.highlightRange(fullHighlights, start, start+endMatch[0], r.group) - h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) - h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) - h.highlightPatterns(fullHighlights, start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) + h.storeRange(start, start+endMatch[0], r.group, r) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + h.highlightPatterns(start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) if curRegion != nil { h.lastRegion = r.parent } else { @@ -205,17 +263,18 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu curRegion = h.lastRegion } } - if nestedRegion || start+startMatch[0] <= h.lastStart || h.lastEnd <= start+startMatch[0] { + if nestedRegion || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { // start at the current line // log.Println("start ...") - h.lastStart = start + startMatch[0] - h.lastEnd = start + lineLen - h.highlightRange(fullHighlights, start+startMatch[0], start+startMatch[1], r.limitGroup) - h.highlightRange(fullHighlights, start+startMatch[1], start+lineLen, r.group) - h.highlightRegions(fullHighlights, start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) - if h.lastStart == 0 || h.lastStart <= start+startMatch[0] { + if h.lastStart == -1 || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + lineLen - 1 h.lastRegion = r } + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r) + h.storeRange(start+startMatch[1], start+lineLen, r.group, r) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) + continue regionLoop } } if curRegion == r { @@ -225,21 +284,21 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // log.Println("... end") h.lastStart = start h.lastEnd = start + endMatch[1] - h.highlightRange(fullHighlights, start, start+endMatch[0], r.group) - h.highlightRange(fullHighlights, start+endMatch[0], start+endMatch[1], r.limitGroup) - h.highlightRegions(fullHighlights, start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + h.storeRange(start, start+endMatch[0], r.group, r) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) if curRegion != nil { h.lastRegion = r.parent } else { h.lastRegion = nil } curRegion = h.lastRegion - h.highlightRegions(fullHighlights, start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) + h.highlightRegions(start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) break } } else if len(startMatches) == 0 && len(endMatches) == 0 { // no start and end found in this region - h.highlightRange(fullHighlights, start, start+lineLen, curRegion.group) + h.storeRange(start, start+lineLen, curRegion.group, r) } } } @@ -248,12 +307,12 @@ func (h *Highlighter) highlightRegions(fullHighlights []Group, start int, lineNu // current region still open // log.Println("...") if curRegion.rules != nil { - h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, curRegion.rules.regions, true) + h.highlightRegions(start, lineNum, line, curRegion, curRegion.rules.regions, true) } if curRegion == h.lastRegion && curRegion.parent != nil { var regions []*region regions = append(regions, curRegion) - h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, regions, true) + h.highlightRegions(start, lineNum, line, curRegion, regions, true) } } } @@ -265,11 +324,20 @@ func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, li return highlights } - h.lastStart = 0 - h.lastEnd = 0 + h.lastStart = -1 + h.lastEnd = -1 + h.storage = h.storage[:0] + + h.highlightRegions(start, lineNum, line, curRegion, h.Def.rules.regions, false) fullHighlights := make([]Group, lineLen) - h.highlightRegions(fullHighlights, start, lineNum, line, curRegion, h.Def.rules.regions, false) + + for _, e := range h.storage { + for i := e.start; i < e.end; i++ { + fullHighlights[i] = e.group + // log.Println("fullHighlights[", i, "]:", e.group) + } + } for i, h := range fullHighlights { if i == 0 || h != fullHighlights[i-1] { From c256f3d5d54133a90d198eabcdb76b0345baa745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:13:14 +0100 Subject: [PATCH 12/17] highlighter: Add capability to remove childrens of already captured groups --- internal/highlight/highlighter.go | 129 +++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 37 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 87d13a10f1..7858c01719 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -24,10 +24,11 @@ type LineStates interface { // highlightStorage is used to store the found ranges type highlightStorage struct { - start int - end int - group Group - region *region + start int + end int + group Group + region *region + children []*highlightStorage } // A Highlighter contains the information needed to highlight a string @@ -89,60 +90,114 @@ func findAllIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) [][]int } func (h *Highlighter) removeRange(start int, end int, removeStart int) { + var children []highlightStorage removeEnd := removeStart for i := removeStart; i < len(h.storage); i++ { - next := h.storage[i] - if start < next.start && next.start < end { - // log.Println("removed: start:", next.start, "end:", next.end, "group:", next.group) + e := h.storage[i] + if start < e.start && e.start < end { + // log.Println("remove: start:", e.start, "end:", e.end, "group:", e.group) removeEnd++ + for childIdx, _ := range h.storage[i].children { + // log.Println("attached child: start:", h.storage[i].children[childIdx].start, "end:", h.storage[i].children[childIdx].end, "group:", h.storage[i].children[childIdx].group) + children = append(children, *(h.storage[i].children[childIdx])) + } } } if removeStart < removeEnd { h.storage = append(h.storage[:removeStart], h.storage[removeEnd:]...) } + + // remove possible children too +childLoop: + for childIdx, _ := range children { + for storageIdx, _ := range h.storage { + if children[childIdx].start == h.storage[storageIdx].start && children[childIdx].end == h.storage[storageIdx].end && children[childIdx].group == h.storage[storageIdx].group && children[childIdx].region == h.storage[storageIdx].region { + // log.Println("remove child: start:", h.storage[storageIdx].start, "end:", h.storage[storageIdx].end, "group:", h.storage[storageIdx].group) + h.storage = append(h.storage[:storageIdx], h.storage[storageIdx+1:]...) + continue childLoop + } + } + } } -func (h *Highlighter) storeRange(start int, end int, group Group, region *region) { +func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isPattern bool) { // log.Println("storeRange: start:", start, "end:", end, "group:", group) + var parent *region + if isPattern { + parent = r + } else if r != nil { + parent = r.parent + } + + updated := false for k, e := range h.storage { - if e.region != nil && region != nil { - if e.region.parent == region.parent { - if region == e.region { - if group == e.group && start == e.end { - // same region, update ... - h.storage[k].end = end - // log.Println("exchanged to: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) - return - } - } else { + if r == e.region && group == e.group && start == e.end { + // same region, update ... + h.storage[k].end = end + // log.Println("exchanged to: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + updated = true + start = h.storage[k].start + } + } + + for k, e := range h.storage { + if e.region != nil && r != nil { + if e.region.parent == parent { + if r != e.region { // sibling regions, search for overlaps ... if start < e.start && end > e.start { // overlap - } else if (start < e.start && end >= e.end) || (start <= e.start && end > e.end) { + } else if start == e.start && end == e.end { + // same match + continue + } else if start <= e.start && end >= e.end { // larger match } else if start >= e.start && end <= e.end { - // smaller or same match + // smaller match return } else { continue } - // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) - h.storage[k] = highlightStorage{start, end, group, region} + if !updated { + // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) + h.storage[k] = highlightStorage{start, end, group, r, nil} - // check and remove follow-ups matching the same - h.removeRange(start, end, k + 1) + // check and remove follow-ups matching the same + h.removeRange(start, end, k+1) + } else { + h.removeRange(start, end, k) + } return } } else { - if region.parent != e.region && start >= e.start && end <= e.end { + if parent != e.region && start >= e.start && end <= e.end { return } } } } - h.storage = append(h.storage, highlightStorage{start, end, group, region}) + if !updated { + h.storage = append(h.storage, highlightStorage{start, end, group, r, nil}) + } + + // add possible child entry + if parent != nil { + storageLoop: + for k, e := range h.storage { + if e.region == parent && e.start < start && end < e.end { + for _, child := range h.storage[k].children { + if child == &(h.storage[len(h.storage)-1]) { + continue storageLoop + } + } + + // log.Println("add child: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + h.storage[k].children = append(h.storage[k].children, &(h.storage[len(h.storage)-1])) + } + } + } } func (h *Highlighter) highlightPatterns(start int, lineNum int, line []byte, curRegion *region) { @@ -162,7 +217,7 @@ func (h *Highlighter) highlightPatterns(start int, lineNum int, line []byte, cur for _, p := range patterns { matches := findAllIndex(p.regex, nil, line) for _, m := range matches { - h.storeRange(start+m[0], start+m[1], p.group, nil) + h.storeRange(start+m[0], start+m[1], p.group, curRegion, true) } } } @@ -227,9 +282,9 @@ regionLoop: h.lastEnd = start + endMatch[1] update = true } - h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r) - h.storeRange(start+startMatch[1], start+endMatch[0], r.group, r) - h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) h.highlightRegions(start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) if samePattern { startIdx += 1 @@ -251,8 +306,8 @@ regionLoop: // log.Println("end < start") h.lastStart = start h.lastEnd = start + endMatch[1] - h.storeRange(start, start+endMatch[0], r.group, r) - h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) h.highlightPatterns(start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) if curRegion != nil { @@ -271,8 +326,8 @@ regionLoop: h.lastEnd = start + lineLen - 1 h.lastRegion = r } - h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r) - h.storeRange(start+startMatch[1], start+lineLen, r.group, r) + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+lineLen, r.group, r, false) h.highlightRegions(start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) continue regionLoop } @@ -284,8 +339,8 @@ regionLoop: // log.Println("... end") h.lastStart = start h.lastEnd = start + endMatch[1] - h.storeRange(start, start+endMatch[0], r.group, r) - h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r) + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) if curRegion != nil { h.lastRegion = r.parent @@ -298,7 +353,7 @@ regionLoop: } } else if len(startMatches) == 0 && len(endMatches) == 0 { // no start and end found in this region - h.storeRange(start, start+lineLen, curRegion.group, r) + h.storeRange(start, start+lineLen, curRegion.group, r, false) } } } From 02115ab54ade315538377dfa68d3ea1b3f56ad96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Thu, 7 Mar 2024 21:10:44 +0100 Subject: [PATCH 13/17] highlighter: Add capability to add already removed groups again --- internal/highlight/highlighter.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 7858c01719..1c4dd11cb8 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -29,6 +29,7 @@ type highlightStorage struct { group Group region *region children []*highlightStorage + pattern bool } // A Highlighter contains the information needed to highlight a string @@ -38,6 +39,7 @@ type Highlighter struct { lastEnd int Def *Def storage []highlightStorage + removed []highlightStorage } // NewHighlighter returns a new highlighter from the given syntax definition @@ -97,6 +99,7 @@ func (h *Highlighter) removeRange(start int, end int, removeStart int) { if start < e.start && e.start < end { // log.Println("remove: start:", e.start, "end:", e.end, "group:", e.group) removeEnd++ + h.removed = append(h.removed, e) for childIdx, _ := range h.storage[i].children { // log.Println("attached child: start:", h.storage[i].children[childIdx].start, "end:", h.storage[i].children[childIdx].end, "group:", h.storage[i].children[childIdx].group) children = append(children, *(h.storage[i].children[childIdx])) @@ -146,7 +149,7 @@ func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isP if r != e.region { // sibling regions, search for overlaps ... if start < e.start && end > e.start { - // overlap + // overlap from left } else if start == e.start && end == e.end { // same match continue @@ -155,13 +158,16 @@ func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isP } else if start >= e.start && end <= e.end { // smaller match return + } else if start > e.start && start < e.end && end > e.end { + // overlap from right + return } else { continue } if !updated { // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) - h.storage[k] = highlightStorage{start, end, group, r, nil} + h.storage[k] = highlightStorage{start, end, group, r, nil, isPattern} // check and remove follow-ups matching the same h.removeRange(start, end, k+1) @@ -179,7 +185,7 @@ func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isP } if !updated { - h.storage = append(h.storage, highlightStorage{start, end, group, r, nil}) + h.storage = append(h.storage, highlightStorage{start, end, group, r, nil, isPattern}) } // add possible child entry @@ -382,9 +388,15 @@ func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, li h.lastStart = -1 h.lastEnd = -1 h.storage = h.storage[:0] + h.removed = h.removed[:0] h.highlightRegions(start, lineNum, line, curRegion, h.Def.rules.regions, false) + // check if entries have been removed by invalid region + for _, e := range h.removed { + h.storeRange(e.start, e.end, e.group, e.region, e.pattern) + } + fullHighlights := make([]Group, lineLen) for _, e := range h.storage { From 1db6fa8d7e93ec2c39fa89bd5a40f10e4b7a97ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Tue, 9 Apr 2024 22:11:51 +0200 Subject: [PATCH 14/17] highlighter: Handle `h.lastRegion` within `h.highlight()` only --- internal/highlight/highlighter.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 1c4dd11cb8..91dc33f92d 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -378,13 +378,14 @@ regionLoop: } } -func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) LineMatch { +func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) (LineMatch, *region) { lineLen := util.CharacterCount(line) // log.Println("highlight: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { - return highlights + return highlights, curRegion } + h.lastRegion = curRegion h.lastStart = -1 h.lastEnd = -1 h.storage = h.storage[:0] @@ -412,7 +413,7 @@ func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, li } } - return highlights + return highlights, h.lastRegion } // HighlightString syntax highlights a string @@ -420,14 +421,16 @@ func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, li // more advanced syntax highlighting. They are optimized for quick rehighlighting of the same // text with minor changes made func (h *Highlighter) HighlightString(input string) []LineMatch { - h.lastRegion = nil lines := strings.Split(input, "\n") var lineMatches []LineMatch + var curState *region for i := 0; i < len(lines); i++ { line := []byte(lines[i]) highlights := make(LineMatch) - lineMatches = append(lineMatches, h.highlight(highlights, 0, i, line, h.lastRegion)) + var match LineMatch + match, curState = h.highlight(highlights, 0, i, line, curState) + lineMatches = append(lineMatches, match) } return lineMatches @@ -436,11 +439,11 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { // Highlight sets the state and matches for each line from startline to endline // It sets all other matches in the buffer to nil to conserve memory func (h *Highlighter) Highlight(input LineStates, startline, endline int) { - h.lastRegion = nil + var curState *region if startline > 0 { input.Lock() if startline-1 < input.LinesNum() { - h.lastRegion = input.State(startline - 1) + curState = input.State(startline - 1) } input.Unlock() } @@ -455,9 +458,10 @@ func (h *Highlighter) Highlight(input LineStates, startline, endline int) { line := input.LineBytes(i) highlights := make(LineMatch) - match := h.highlight(highlights, 0, i, line, h.lastRegion) + var match LineMatch + match, curState = h.highlight(highlights, 0, i, line, curState) - input.SetState(i, h.lastRegion) + input.SetState(i, curState) input.SetMatch(i, match) input.Unlock() } @@ -471,13 +475,14 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { line := input.LineBytes(lineN) highlights := make(LineMatch) - h.lastRegion = nil + var curState *region if lineN > 0 { - h.lastRegion = input.State(lineN - 1) + curState = input.State(lineN - 1) } - match := h.highlight(highlights, 0, lineN, line, h.lastRegion) + var match LineMatch + match, curState = h.highlight(highlights, 0, lineN, line, curState) - input.SetState(lineN, h.lastRegion) + input.SetState(lineN, curState) input.SetMatch(lineN, match) } From 5477395537ab86e5dfefd73bfa76bb792779f885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Thu, 11 Apr 2024 21:24:24 +0200 Subject: [PATCH 15/17] highlighter: Properly handle rehighlighting within `Highlight()` Co-authored-by: Dmitry Maluka --- internal/highlight/highlighter.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 91dc33f92d..8699be69d0 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -436,8 +436,10 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { return lineMatches } -// Highlight sets the state and matches for each line from startline to endline -// It sets all other matches in the buffer to nil to conserve memory +// Highlight sets the state and matches for each line from startline to endline, +// and also for some amount of lines after endline, until it detects a line +// whose state does not change, which means that the lines after it do not change +// their highlighting and therefore do not need to be updated. func (h *Highlighter) Highlight(input LineStates, startline, endline int) { var curState *region if startline > 0 { @@ -448,7 +450,7 @@ func (h *Highlighter) Highlight(input LineStates, startline, endline int) { input.Unlock() } - for i := startline; i <= endline; i++ { + for i := startline; ; i++ { input.Lock() if i >= input.LinesNum() { input.Unlock() @@ -461,9 +463,18 @@ func (h *Highlighter) Highlight(input LineStates, startline, endline int) { var match LineMatch match, curState = h.highlight(highlights, 0, i, line, curState) + var lastState *region + if i >= endline { + lastState = input.State(i) + } + input.SetState(i, curState) input.SetMatch(i, match) input.Unlock() + + if i >= endline && curState == lastState { + break + } } } From f56e627a89e198d239b4ac2a0e3fadbbacb1cc47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Sun, 16 Mar 2025 12:52:23 +0100 Subject: [PATCH 16/17] highlighter: Simplify `samePattern` logic --- internal/highlight/highlighter.go | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go index 8699be69d0..a87ad8210b 100644 --- a/internal/highlight/highlighter.go +++ b/internal/highlight/highlighter.go @@ -249,7 +249,7 @@ regionLoop: } startMatches := findAllIndex(r.start, r.skip, line) endMatches := findAllIndex(r.end, r.skip, line) - samePattern := false + samePattern := r.start.String() == r.end.String() startLoop: for startIdx := 0; startIdx < len(startMatches); startIdx++ { // log.Println("startIdx:", startIdx, "of", len(startMatches)) @@ -258,22 +258,13 @@ regionLoop: // log.Println("startIdx:", startIdx, "of", len(startMatches), "/ endIdx:", endIdx, "of", len(endMatches), "/ h.lastStart:", h.lastStart, "/ h.lastEnd:", h.lastEnd) endMatch := endMatches[endIdx] if startMatch[0] == endMatch[0] { - // start and end are the same (pattern) - // log.Println("start == end") - samePattern = true - if len(startMatches) == len(endMatches) { - // special case in the moment both are the same + if samePattern { + // start and end are the same + // log.Println("start == end") if curRegion == r { - if len(startMatches) > 1 { - // end < start - continue startLoop - } else if len(startMatches) > 0 { - // ... end - startIdx = len(startMatches) - continue startLoop - } + continue startLoop } else { - // start ... or start < end + continue } } } else if startMatch[1] <= endMatch[0] { @@ -339,7 +330,10 @@ regionLoop: } } if curRegion == r { - if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && (len(startMatches) == len(endMatches))) { + if len(startMatches) == 0 && len(endMatches) == 0 { + // no start and end found in this region + h.storeRange(start, start+lineLen, curRegion.group, r, false) + } else if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && len(endMatches) > 0) { for _, endMatch := range endMatches { // end at the current line // log.Println("... end") @@ -357,9 +351,6 @@ regionLoop: h.highlightRegions(start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) break } - } else if len(startMatches) == 0 && len(endMatches) == 0 { - // no start and end found in this region - h.storeRange(start, start+lineLen, curRegion.group, r, false) } } } From db7581ce29a54107e2130a39f8d9d6a5d8fdcbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6ran=20Karl?= <3951388+JoeKar@users.noreply.github.com> Date: Tue, 25 Mar 2025 23:57:03 +0100 Subject: [PATCH 17/17] highlighter: Add basic syntax highlighting test --- internal/highlight/highlighter_test.go | 128 +++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 internal/highlight/highlighter_test.go diff --git a/internal/highlight/highlighter_test.go b/internal/highlight/highlighter_test.go new file mode 100644 index 0000000000..03706175b7 --- /dev/null +++ b/internal/highlight/highlighter_test.go @@ -0,0 +1,128 @@ +package highlight + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const syntax = ` +filetype: test +detect: + filename: "test" +rules: + - type: "\\b(one|two|three)\\b" + - constant.string: + start: "\"" + end: "\"" + - constant.string: + start: "'" + end: "'" + - comment: + start: "//" + end: "$" + rules: + - todo: "(TODO):?" + - comment: + start: "/\\*" + end: "\\*/" + rules: + - todo: "(TODO):?" +` + +const content = ` +one two three +// comment +one +two +three +/* +multi +line +// comment +with +TODO +*/ +"string" +one two three +" +multi +line +str'ng +" +one two three +'string' +one two three +// ' +one "string" two /*rule*/ three //all +/* " */ +` + +var expectation = []map[int]string{ + {}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "comment"}, + {0: "type"}, + {0: "type"}, + {0: "type"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "todo"}, + {0: "comment"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "comment"}, + {0: "type", 3: "", 4: "constant.string", 12: "", 13: "type", 16: "", 17: "comment", 25: "", 26: "type", 31: "", 32: "comment"}, + {0: "comment"}, + {}, +} + +func TestHighlightString(t *testing.T) { + header, err := MakeHeaderYaml([]byte(syntax)) + if !assert.NoError(t, err) { + return + } + + file, err := ParseFile([]byte(syntax)) + if !assert.NoError(t, err) { + return + } + + def, err := ParseDef(file, header) + if !assert.NoError(t, err) { + return + } + + highlighter := NewHighlighter(def) + matches := highlighter.HighlightString(content) + result := assert.Equal(t, len(expectation), len(matches)) + if !result { + return + } + + for i, m := range matches { + result = assert.Equal(t, len(expectation[i]), len(m)) + if !result { + return + } + actual := map[int]string{} + for k, g := range m { + actual[k] = g.String() + } + result := assert.Equal(t, expectation[i], actual, i) + if !result { + return + } + } +}