diff --git a/internal/action/infocomplete.go b/internal/action/infocomplete.go index b8ee879a30..783132fc9d 100644 --- a/internal/action/infocomplete.go +++ b/internal/action/infocomplete.go @@ -7,8 +7,8 @@ import ( "github.com/zyedidia/micro/v2/internal/buffer" "github.com/zyedidia/micro/v2/internal/config" + "github.com/zyedidia/micro/v2/internal/highlight" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" ) // This file is meant (for now) for autocompletion in command mode, not diff --git a/internal/buffer/buffer.go b/internal/buffer/buffer.go index 4226d972cd..ba9b4a6b20 100644 --- a/internal/buffer/buffer.go +++ b/internal/buffer/buffer.go @@ -21,10 +21,10 @@ import ( dmp "github.com/sergi/go-diff/diffmatchpatch" "github.com/zyedidia/micro/v2/internal/config" + "github.com/zyedidia/micro/v2/internal/highlight" ulua "github.com/zyedidia/micro/v2/internal/lua" "github.com/zyedidia/micro/v2/internal/screen" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" "golang.org/x/text/encoding" "golang.org/x/text/encoding/htmlindex" "golang.org/x/text/encoding/unicode" @@ -150,11 +150,7 @@ func (b *SharedBuffer) MarkModified(start, end int) { end = util.Clamp(end, 0, len(b.lines)-1) if b.Settings["syntax"].(bool) && b.SyntaxDef != nil { - l := -1 - for i := start; i <= end; i++ { - l = util.Max(b.Highlighter.ReHighlightStates(b, i), l) - } - b.Highlighter.HighlightMatches(b, start, l) + b.Highlighter.Highlight(b, start, end) } for i := start; i <= end; i++ { @@ -960,8 +956,7 @@ func (b *Buffer) UpdateRules() { b.Highlighter = highlight.NewHighlighter(b.SyntaxDef) if b.Settings["syntax"].(bool) { go func() { - b.Highlighter.HighlightStates(b) - b.Highlighter.HighlightMatches(b, 0, b.End().Y) + b.Highlighter.Highlight(b, 0, b.End().Y) screen.Redraw() }() } diff --git a/internal/buffer/line_array.go b/internal/buffer/line_array.go index b65213b805..97ced52122 100644 --- a/internal/buffer/line_array.go +++ b/internal/buffer/line_array.go @@ -6,8 +6,8 @@ import ( "io" "sync" + "github.com/zyedidia/micro/v2/internal/highlight" "github.com/zyedidia/micro/v2/internal/util" - "github.com/zyedidia/micro/v2/pkg/highlight" ) // Finds the byte index of the nth rune in a byte slice diff --git a/internal/highlight/highlighter.go b/internal/highlight/highlighter.go new file mode 100644 index 0000000000..a87ad8210b --- /dev/null +++ b/internal/highlight/highlighter.go @@ -0,0 +1,490 @@ +package highlight + +import ( + // "log" + "regexp" + "strings" + + "github.com/zyedidia/micro/v2/internal/util" +) + +// A State represents the region at the end of a line +type State *region + +// LineStates is an interface for a buffer-like object which can also store the states and matches for every line +type LineStates interface { + LineBytes(n int) []byte + LinesNum() int + State(lineN int) State + SetState(lineN int, s State) + SetMatch(lineN int, m LineMatch) + Lock() + Unlock() +} + +// highlightStorage is used to store the found ranges +type highlightStorage struct { + start int + end int + group Group + region *region + children []*highlightStorage + pattern bool +} + +// A Highlighter contains the information needed to highlight a string +type Highlighter struct { + lastRegion *region + lastStart int + lastEnd int + Def *Def + storage []highlightStorage + removed []highlightStorage +} + +// NewHighlighter returns a new highlighter from the given syntax definition +func NewHighlighter(def *Def) *Highlighter { + h := new(Highlighter) + h.Def = def + return h +} + +// LineMatch represents the syntax highlighting matches for one line. Each index where the coloring is changed is marked with that +// color's group (represented as one byte) +type LineMatch map[int]Group + +func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { + var strbytes []byte + if skip != nil { + strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { + res := make([]byte, util.CharacterCount(match)) + return res + }) + } else { + strbytes = str + } + + match := regex.FindIndex(strbytes) + if match == nil { + return nil + } + // return []int{match.Index, match.Index + match.Length} + return []int{util.RunePos(str, match[0]), util.RunePos(str, match[1])} +} + +func findAllIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) [][]int { + var strbytes []byte + if skip != nil { + strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { + res := make([]byte, util.CharacterCount(match)) + return res + }) + } else { + strbytes = str + } + + matches := regex.FindAllIndex(strbytes, -1) + for i, m := range matches { + matches[i][0] = util.RunePos(str, m[0]) + matches[i][1] = util.RunePos(str, m[1]) + } + return matches +} + +func (h *Highlighter) removeRange(start int, end int, removeStart int) { + var children []highlightStorage + removeEnd := removeStart + for i := removeStart; i < len(h.storage); i++ { + e := h.storage[i] + if start < e.start && e.start < end { + // log.Println("remove: start:", e.start, "end:", e.end, "group:", e.group) + removeEnd++ + h.removed = append(h.removed, e) + for childIdx, _ := range h.storage[i].children { + // log.Println("attached child: start:", h.storage[i].children[childIdx].start, "end:", h.storage[i].children[childIdx].end, "group:", h.storage[i].children[childIdx].group) + children = append(children, *(h.storage[i].children[childIdx])) + } + } + } + if removeStart < removeEnd { + h.storage = append(h.storage[:removeStart], h.storage[removeEnd:]...) + } + + // remove possible children too +childLoop: + for childIdx, _ := range children { + for storageIdx, _ := range h.storage { + if children[childIdx].start == h.storage[storageIdx].start && children[childIdx].end == h.storage[storageIdx].end && children[childIdx].group == h.storage[storageIdx].group && children[childIdx].region == h.storage[storageIdx].region { + // log.Println("remove child: start:", h.storage[storageIdx].start, "end:", h.storage[storageIdx].end, "group:", h.storage[storageIdx].group) + h.storage = append(h.storage[:storageIdx], h.storage[storageIdx+1:]...) + continue childLoop + } + } + } +} + +func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isPattern bool) { + // log.Println("storeRange: start:", start, "end:", end, "group:", group) + var parent *region + if isPattern { + parent = r + } else if r != nil { + parent = r.parent + } + + updated := false + for k, e := range h.storage { + if r == e.region && group == e.group && start == e.end { + // same region, update ... + h.storage[k].end = end + // log.Println("exchanged to: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + updated = true + start = h.storage[k].start + } + } + + for k, e := range h.storage { + if e.region != nil && r != nil { + if e.region.parent == parent { + if r != e.region { + // sibling regions, search for overlaps ... + if start < e.start && end > e.start { + // overlap from left + } else if start == e.start && end == e.end { + // same match + continue + } else if start <= e.start && end >= e.end { + // larger match + } else if start >= e.start && end <= e.end { + // smaller match + return + } else if start > e.start && start < e.end && end > e.end { + // overlap from right + return + } else { + continue + } + + if !updated { + // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) + h.storage[k] = highlightStorage{start, end, group, r, nil, isPattern} + + // check and remove follow-ups matching the same + h.removeRange(start, end, k+1) + } else { + h.removeRange(start, end, k) + } + return + } + } else { + if parent != e.region && start >= e.start && end <= e.end { + return + } + } + } + } + + if !updated { + h.storage = append(h.storage, highlightStorage{start, end, group, r, nil, isPattern}) + } + + // add possible child entry + if parent != nil { + storageLoop: + for k, e := range h.storage { + if e.region == parent && e.start < start && end < e.end { + for _, child := range h.storage[k].children { + if child == &(h.storage[len(h.storage)-1]) { + continue storageLoop + } + } + + // log.Println("add child: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + h.storage[k].children = append(h.storage[k].children, &(h.storage[len(h.storage)-1])) + } + } + } +} + +func (h *Highlighter) highlightPatterns(start int, lineNum int, line []byte, curRegion *region) { + lineLen := util.CharacterCount(line) + // log.Println("highlightPatterns: lineNum:", lineNum, "start:", start, "line:", string(line)) + if lineLen == 0 { + return + } + + var patterns []*pattern + if curRegion == nil { + patterns = h.Def.rules.patterns + } else { + patterns = curRegion.rules.patterns + } + + for _, p := range patterns { + matches := findAllIndex(p.regex, nil, line) + for _, m := range matches { + h.storeRange(start+m[0], start+m[1], p.group, curRegion, true) + } + } +} + +func (h *Highlighter) highlightRegions(start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { + lineLen := util.CharacterCount(line) + // log.Println("highlightRegions: lineNum:", lineNum, "start:", start, "line:", string(line)) + if lineLen == 0 { + return + } + + if nestedRegion { + h.highlightPatterns(start, lineNum, line, curRegion) + } else { + h.highlightPatterns(start, lineNum, line, nil) + } + +regionLoop: + for _, r := range regions { + // log.Println("r.start:", r.start.String(), "r.end:", r.end.String()) + if !nestedRegion && curRegion != nil && curRegion != r { + continue + } + startMatches := findAllIndex(r.start, r.skip, line) + endMatches := findAllIndex(r.end, r.skip, line) + samePattern := r.start.String() == r.end.String() + startLoop: + for startIdx := 0; startIdx < len(startMatches); startIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches)) + startMatch := startMatches[startIdx] + for endIdx := 0; endIdx < len(endMatches); endIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches), "/ endIdx:", endIdx, "of", len(endMatches), "/ h.lastStart:", h.lastStart, "/ h.lastEnd:", h.lastEnd) + endMatch := endMatches[endIdx] + if startMatch[0] == endMatch[0] { + if samePattern { + // start and end are the same + // log.Println("start == end") + if curRegion == r { + continue startLoop + } else { + continue + } + } + } else if startMatch[1] <= endMatch[0] { + if !nestedRegion && h.lastStart < start+startMatch[0] && start+startMatch[0] < h.lastEnd { + continue + } + // start and end at the current line + // log.Println("start < end") + update := false + if h.lastStart == -1 || h.lastStart < start+endMatch[1] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + endMatch[1] + update = true + } + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) + if samePattern { + startIdx += 1 + } + if update { + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + continue startLoop + } else if endMatch[1] <= startMatch[0] { + if start+endMatch[0] < h.lastEnd || curRegion == nil { + continue + } + // start and end at the current line, but switched + // log.Println("end < start") + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + h.highlightPatterns(start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + } + if nestedRegion || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { + // start at the current line + // log.Println("start ...") + if h.lastStart == -1 || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + lineLen - 1 + h.lastRegion = r + } + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+lineLen, r.group, r, false) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) + continue regionLoop + } + } + if curRegion == r { + if len(startMatches) == 0 && len(endMatches) == 0 { + // no start and end found in this region + h.storeRange(start, start+lineLen, curRegion.group, r, false) + } else if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && len(endMatches) > 0) { + for _, endMatch := range endMatches { + // end at the current line + // log.Println("... end") + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + h.highlightRegions(start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) + break + } + } + } + } + + if curRegion != nil && !nestedRegion { + // current region still open + // log.Println("...") + if curRegion.rules != nil { + h.highlightRegions(start, lineNum, line, curRegion, curRegion.rules.regions, true) + } + if curRegion == h.lastRegion && curRegion.parent != nil { + var regions []*region + regions = append(regions, curRegion) + h.highlightRegions(start, lineNum, line, curRegion, regions, true) + } + } +} + +func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) (LineMatch, *region) { + lineLen := util.CharacterCount(line) + // log.Println("highlight: lineNum:", lineNum, "start:", start, "line:", string(line)) + if lineLen == 0 { + return highlights, curRegion + } + + h.lastRegion = curRegion + h.lastStart = -1 + h.lastEnd = -1 + h.storage = h.storage[:0] + h.removed = h.removed[:0] + + h.highlightRegions(start, lineNum, line, curRegion, h.Def.rules.regions, false) + + // check if entries have been removed by invalid region + for _, e := range h.removed { + h.storeRange(e.start, e.end, e.group, e.region, e.pattern) + } + + fullHighlights := make([]Group, lineLen) + + for _, e := range h.storage { + for i := e.start; i < e.end; i++ { + fullHighlights[i] = e.group + // log.Println("fullHighlights[", i, "]:", e.group) + } + } + + for i, h := range fullHighlights { + if i == 0 || h != fullHighlights[i-1] { + highlights[i] = h + } + } + + return highlights, h.lastRegion +} + +// HighlightString syntax highlights a string +// Use this function for simple syntax highlighting and use the other functions for +// more advanced syntax highlighting. They are optimized for quick rehighlighting of the same +// text with minor changes made +func (h *Highlighter) HighlightString(input string) []LineMatch { + lines := strings.Split(input, "\n") + var lineMatches []LineMatch + var curState *region + + for i := 0; i < len(lines); i++ { + line := []byte(lines[i]) + highlights := make(LineMatch) + var match LineMatch + match, curState = h.highlight(highlights, 0, i, line, curState) + lineMatches = append(lineMatches, match) + } + + return lineMatches +} + +// Highlight sets the state and matches for each line from startline to endline, +// and also for some amount of lines after endline, until it detects a line +// whose state does not change, which means that the lines after it do not change +// their highlighting and therefore do not need to be updated. +func (h *Highlighter) Highlight(input LineStates, startline, endline int) { + var curState *region + if startline > 0 { + input.Lock() + if startline-1 < input.LinesNum() { + curState = input.State(startline - 1) + } + input.Unlock() + } + + for i := startline; ; i++ { + input.Lock() + if i >= input.LinesNum() { + input.Unlock() + break + } + + line := input.LineBytes(i) + highlights := make(LineMatch) + + var match LineMatch + match, curState = h.highlight(highlights, 0, i, line, curState) + + var lastState *region + if i >= endline { + lastState = input.State(i) + } + + input.SetState(i, curState) + input.SetMatch(i, match) + input.Unlock() + + if i >= endline && curState == lastState { + break + } + } +} + +// ReHighlightLine will rehighlight the state and match for a single line +func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { + input.Lock() + defer input.Unlock() + + line := input.LineBytes(lineN) + highlights := make(LineMatch) + + var curState *region + if lineN > 0 { + curState = input.State(lineN - 1) + } + + var match LineMatch + match, curState = h.highlight(highlights, 0, lineN, line, curState) + + input.SetState(lineN, curState) + input.SetMatch(lineN, match) +} diff --git a/internal/highlight/highlighter_test.go b/internal/highlight/highlighter_test.go new file mode 100644 index 0000000000..03706175b7 --- /dev/null +++ b/internal/highlight/highlighter_test.go @@ -0,0 +1,128 @@ +package highlight + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const syntax = ` +filetype: test +detect: + filename: "test" +rules: + - type: "\\b(one|two|three)\\b" + - constant.string: + start: "\"" + end: "\"" + - constant.string: + start: "'" + end: "'" + - comment: + start: "//" + end: "$" + rules: + - todo: "(TODO):?" + - comment: + start: "/\\*" + end: "\\*/" + rules: + - todo: "(TODO):?" +` + +const content = ` +one two three +// comment +one +two +three +/* +multi +line +// comment +with +TODO +*/ +"string" +one two three +" +multi +line +str'ng +" +one two three +'string' +one two three +// ' +one "string" two /*rule*/ three //all +/* " */ +` + +var expectation = []map[int]string{ + {}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "comment"}, + {0: "type"}, + {0: "type"}, + {0: "type"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "comment"}, + {0: "todo"}, + {0: "comment"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "constant.string"}, + {0: "type", 3: "", 4: "type", 7: "", 8: "type"}, + {0: "comment"}, + {0: "type", 3: "", 4: "constant.string", 12: "", 13: "type", 16: "", 17: "comment", 25: "", 26: "type", 31: "", 32: "comment"}, + {0: "comment"}, + {}, +} + +func TestHighlightString(t *testing.T) { + header, err := MakeHeaderYaml([]byte(syntax)) + if !assert.NoError(t, err) { + return + } + + file, err := ParseFile([]byte(syntax)) + if !assert.NoError(t, err) { + return + } + + def, err := ParseDef(file, header) + if !assert.NoError(t, err) { + return + } + + highlighter := NewHighlighter(def) + matches := highlighter.HighlightString(content) + result := assert.Equal(t, len(expectation), len(matches)) + if !result { + return + } + + for i, m := range matches { + result = assert.Equal(t, len(expectation[i]), len(m)) + if !result { + return + } + actual := map[int]string{} + for k, g := range m { + actual[k] = g.String() + } + result := assert.Equal(t, expectation[i], actual, i) + if !result { + return + } + } +} diff --git a/pkg/highlight/parser.go b/internal/highlight/parser.go similarity index 100% rename from pkg/highlight/parser.go rename to internal/highlight/parser.go diff --git a/internal/util/util.go b/internal/util/util.go index f2cb2a99d9..0405e37d24 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -171,6 +171,16 @@ func SliceStartStr(str string, index int) string { return str[:totalSize] } +// SliceStartEnd combines SliceStart and SliceEnd into one +func SliceStartEnd(slc []byte, start int, end int) []byte { + return SliceEnd(SliceStart(slc, end), start) +} + +// SliceStartEndStr is the same as SliceStartEnd but for strings +func SliceStartEndStr(str string, start int, end int) string { + return SliceEndStr(SliceStartStr(str, end), start) +} + // SliceVisualEnd will take a byte slice and slice off the start // up to a given visual index. If the index is in the middle of a // rune the number of visual columns into the rune will be returned diff --git a/pkg/highlight/highlighter.go b/pkg/highlight/highlighter.go deleted file mode 100644 index a13a72610d..0000000000 --- a/pkg/highlight/highlighter.go +++ /dev/null @@ -1,408 +0,0 @@ -package highlight - -import ( - "regexp" - "strings" -) - -func sliceStart(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[totalSize:] - } - - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - - return slc[totalSize:] -} - -func sliceEnd(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[:totalSize] - } - - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - - return slc[:totalSize] -} - -// RunePos returns the rune index of a given byte index -// This could cause problems if the byte index is between code points -func runePos(p int, str []byte) int { - if p < 0 { - return 0 - } - if p >= len(str) { - return CharacterCount(str) - } - return CharacterCount(str[:p]) -} - -// A State represents the region at the end of a line -type State *region - -// LineStates is an interface for a buffer-like object which can also store the states and matches for every line -type LineStates interface { - LineBytes(n int) []byte - LinesNum() int - State(lineN int) State - SetState(lineN int, s State) - SetMatch(lineN int, m LineMatch) - Lock() - Unlock() -} - -// A Highlighter contains the information needed to highlight a string -type Highlighter struct { - lastRegion *region - Def *Def -} - -// NewHighlighter returns a new highlighter from the given syntax definition -func NewHighlighter(def *Def) *Highlighter { - h := new(Highlighter) - h.Def = def - return h -} - -// LineMatch represents the syntax highlighting matches for one line. Each index where the coloring is changed is marked with that -// color's group (represented as one byte) -type LineMatch map[int]Group - -func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { - var strbytes []byte - if skip != nil { - strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { - res := make([]byte, CharacterCount(match)) - return res - }) - } else { - strbytes = str - } - - match := regex.FindIndex(strbytes) - if match == nil { - return nil - } - // return []int{match.Index, match.Index + match.Length} - return []int{runePos(match[0], str), runePos(match[1], str)} -} - -func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { - matches := regex.FindAllIndex(str, -1) - for i, m := range matches { - matches[i][0] = runePos(m[0], str) - matches[i][1] = runePos(m[1], str) - } - return matches -} - -func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) - if start == 0 { - if !statesOnly { - if _, ok := highlights[0]; !ok { - highlights[0] = curRegion.group - } - } - } - - var firstRegion *region - firstLoc := []int{lineLen, 0} - searchNesting := true - endLoc := findIndex(curRegion.end, curRegion.skip, line) - if endLoc != nil { - if start == endLoc[0] { - searchNesting = false - } else { - firstLoc = endLoc - } - } - if searchNesting { - for _, r := range curRegion.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r - } - } - } - } - if firstRegion != nil && firstLoc[0] != lineLen { - if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup - } - h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) - return highlights - } - - if !statesOnly { - fullHighlights := make([]Group, lineLen) - for i := 0; i < len(fullHighlights); i++ { - fullHighlights[i] = curRegion.group - } - - if searchNesting { - for _, p := range curRegion.rules.patterns { - if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - if (endLoc == nil) || (m[0] < endLoc[0]) { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } - } - } - } - } - } - for i, h := range fullHighlights { - if i == 0 || h != fullHighlights[i-1] { - highlights[start+i] = h - } - } - } - - loc := endLoc - if loc != nil { - if !statesOnly { - highlights[start+loc[0]] = curRegion.limitGroup - } - if curRegion.parent == nil { - if !statesOnly { - highlights[start+loc[1]] = 0 - } - h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly) - return highlights - } - if !statesOnly { - highlights[start+loc[1]] = curRegion.parent.group - } - h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly) - return highlights - } - - if canMatchEnd { - h.lastRegion = curRegion - } - - return highlights -} - -func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) - if lineLen == 0 { - if canMatchEnd { - h.lastRegion = nil - } - return highlights - } - - var firstRegion *region - firstLoc := []int{lineLen, 0} - for _, r := range h.Def.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r - } - } - } - if firstRegion != nil && firstLoc[0] != lineLen { - if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup - } - h.highlightEmptyRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) - return highlights - } - - if statesOnly { - if canMatchEnd { - h.lastRegion = nil - } - - return highlights - } - - fullHighlights := make([]Group, len(line)) - for _, p := range h.Def.rules.patterns { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } - } - } - for i, h := range fullHighlights { - if i == 0 || h != fullHighlights[i-1] { - // if _, ok := highlights[start+i]; !ok { - highlights[start+i] = h - // } - } - } - - if canMatchEnd { - h.lastRegion = nil - } - - return highlights -} - -// HighlightString syntax highlights a string -// Use this function for simple syntax highlighting and use the other functions for -// more advanced syntax highlighting. They are optimized for quick rehighlighting of the same -// text with minor changes made -func (h *Highlighter) HighlightString(input string) []LineMatch { - lines := strings.Split(input, "\n") - var lineMatches []LineMatch - - for i := 0; i < len(lines); i++ { - line := []byte(lines[i]) - highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, true, i, line, false)) - } else { - lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, true, i, line, h.lastRegion, false)) - } - } - - return lineMatches -} - -// HighlightStates correctly sets all states for the buffer -func (h *Highlighter) HighlightStates(input LineStates) { - for i := 0; ; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - // highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) - } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) - } - - curState := h.lastRegion - - input.SetState(i, curState) - input.Unlock() - } -} - -// HighlightMatches sets the matches for each line from startline to endline -// It sets all other matches in the buffer to nil to conserve memory -// This assumes that all the states are set correctly -func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int) { - for i := startline; i <= endline; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - highlights := make(LineMatch) - - var match LineMatch - if i == 0 || input.State(i-1) == nil { - match = h.highlightEmptyRegion(highlights, 0, true, i, line, false) - } else { - match = h.highlightRegion(highlights, 0, true, i, line, input.State(i-1), false) - } - - input.SetMatch(i, match) - input.Unlock() - } -} - -// ReHighlightStates will scan down from `startline` and set the appropriate end of line state -// for each line until it comes across a line whose state does not change -// returns the number of the final line -func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { - // lines := input.LineData() - - h.lastRegion = nil - if startline > 0 { - input.Lock() - if startline-1 < input.LinesNum() { - h.lastRegion = input.State(startline - 1) - } - input.Unlock() - } - for i := startline; ; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - // highlights := make(LineMatch) - - // var match LineMatch - if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) - } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) - } - curState := h.lastRegion - lastState := input.State(i) - - input.SetState(i, curState) - input.Unlock() - - if curState == lastState { - return i - } - } - - return input.LinesNum() - 1 -} - -// ReHighlightLine will rehighlight the state and match for a single line -func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { - input.Lock() - defer input.Unlock() - - line := input.LineBytes(lineN) - highlights := make(LineMatch) - - h.lastRegion = nil - if lineN > 0 { - h.lastRegion = input.State(lineN - 1) - } - - var match LineMatch - if lineN == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, true, lineN, line, false) - } else { - match = h.highlightRegion(highlights, 0, true, lineN, line, h.lastRegion, false) - } - curState := h.lastRegion - - input.SetMatch(lineN, match) - input.SetState(lineN, curState) -} diff --git a/pkg/highlight/unicode.go b/pkg/highlight/unicode.go deleted file mode 100644 index a18118a6ca..0000000000 --- a/pkg/highlight/unicode.go +++ /dev/null @@ -1,85 +0,0 @@ -package highlight - -import ( - "unicode" - "unicode/utf8" -) - -var minMark = rune(unicode.Mark.R16[0].Lo) - -func isMark(r rune) bool { - // Fast path - if r < minMark { - return false - } - return unicode.In(r, unicode.Mark) -} - -// DecodeCharacter returns the next character from an array of bytes -// A character is a rune along with any accompanying combining runes -func DecodeCharacter(b []byte) (rune, []rune, int) { - r, size := utf8.DecodeRune(b) - b = b[size:] - c, s := utf8.DecodeRune(b) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - b = b[s:] - c, s = utf8.DecodeRune(b) - } - - return r, combc, size -} - -// DecodeCharacterInString returns the next character from a string -// A character is a rune along with any accompanying combining runes -func DecodeCharacterInString(str string) (rune, []rune, int) { - r, size := utf8.DecodeRuneInString(str) - str = str[size:] - c, s := utf8.DecodeRuneInString(str) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - str = str[s:] - c, s = utf8.DecodeRuneInString(str) - } - - return r, combc, size -} - -// CharacterCount returns the number of characters in a byte array -// Similar to utf8.RuneCount but for unicode characters -func CharacterCount(b []byte) int { - s := 0 - - for len(b) > 0 { - r, size := utf8.DecodeRune(b) - if !isMark(r) { - s++ - } - - b = b[size:] - } - - return s -} - -// CharacterCount returns the number of characters in a string -// Similar to utf8.RuneCountInString but for unicode characters -func CharacterCountInString(str string) int { - s := 0 - - for _, r := range str { - if !isMark(r) { - s++ - } - } - - return s -}