NeonXP/micro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Better unicode support in highlight

Browse source
This commit is contained in:
Zachary Yedidia 2017-03-27 19:35:28 -04:00
parent bde48c051a
commit c24f75999a
4 changed files with 92 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/micro/cellview.go
View file

@ -2,7 +2,6 @@ package main
import ( import (
"github.com/mattn/go-runewidth" "github.com/mattn/go-runewidth"
"github.com/zyedidia/micro/cmd/micro/highlight"
"github.com/zyedidia/tcell" "github.com/zyedidia/tcell"
) )
@ -24,7 +23,7 @@ func visualToCharPos(visualIndex int, lineN int, str string, buf *Buffer, tabsiz
// width := StringWidth(str[:i], tabsize) // width := StringWidth(str[:i], tabsize)
if group, ok := buf.Match(lineN)[charPos]; ok { if group, ok := buf.Match(lineN)[charPos]; ok {
s := GetColor(highlight.GetGroup(group)) s := GetColor(group.String())
style = &s style = &s
} }
@ -123,7 +122,7 @@ func (c *CellView) Draw(buf *Buffer, top, height, left, width int) {
break break
} }
if group, ok := buf.Match(lineN)[colN]; ok { if group, ok := buf.Match(lineN)[colN]; ok {
curStyle = GetColor(highlight.GetGroup(group)) curStyle = GetColor(group.String())
} }
char := line[colN] char := line[colN]
@ -186,7 +185,7 @@ func (c *CellView) Draw(buf *Buffer, top, height, left, width int) {
} }
if group, ok := buf.Match(lineN)[len(line)]; ok { if group, ok := buf.Match(lineN)[len(line)]; ok {
curStyle = GetColor(highlight.GetGroup(group)) curStyle = GetColor(group.String())
} }
// newline // newline

5
cmd/micro/highlight/ftdetect.go
View file

@ -1,5 +1,8 @@
package highlight package highlight
// DetectFiletype will use the list of syntax definitions provided and the filename and first line of the file
// to determine the filetype of the file
// It will return the corresponding syntax definition for the filetype
func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def { func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def {
for _, d := range defs { for _, d := range defs {
if d.ftdetect[0].MatchString(filename) { if d.ftdetect[0].MatchString(filename) {
@ -14,6 +17,6 @@ func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def {
emptyDef := new(Def) emptyDef := new(Def)
emptyDef.FileType = "Unknown" emptyDef.FileType = "Unknown"
emptyDef.rules = new(Rules) emptyDef.rules = new(rules)
return emptyDef return emptyDef
} }

97
cmd/micro/highlight/highlighter.go
View file

@ -34,7 +34,7 @@ func combineLineMatch(src, dst LineMatch) LineMatch {
} }
// A State represents the region at the end of a line // A State represents the region at the end of a line
type State *Region type State *region
// LineStates is an interface for a buffer-like object which can also store the states and matches for every line // LineStates is an interface for a buffer-like object which can also store the states and matches for every line
type LineStates interface { type LineStates interface {
@ -47,22 +47,22 @@ type LineStates interface {
// A Highlighter contains the information needed to highlight a string // A Highlighter contains the information needed to highlight a string
type Highlighter struct { type Highlighter struct {
lastRegion *Region lastRegion *region
def *Def Def *Def
} }
// NewHighlighter returns a new highlighter from the given syntax definition // NewHighlighter returns a new highlighter from the given syntax definition
func NewHighlighter(def *Def) *Highlighter { func NewHighlighter(def *Def) *Highlighter {
h := new(Highlighter) h := new(Highlighter)
h.def = def h.Def = def
return h return h
} }
// LineMatch represents the syntax highlighting matches for one line. Each index where the coloring is changed is marked with that // LineMatch represents the syntax highlighting matches for one line. Each index where the coloring is changed is marked with that
// color's group (represented as one byte) // color's group (represented as one byte)
type LineMatch map[int]uint8 type LineMatch map[int]Group
func findIndex(regex *regexp2.Regexp, str []byte, canMatchStart, canMatchEnd bool) []int { func findIndex(regex *regexp2.Regexp, str []rune, canMatchStart, canMatchEnd bool) []int {
regexStr := regex.String() regexStr := regex.String()
if strings.Contains(regexStr, "^") { if strings.Contains(regexStr, "^") {
if !canMatchStart { if !canMatchStart {
@ -79,9 +79,10 @@ func findIndex(regex *regexp2.Regexp, str []byte, canMatchStart, canMatchEnd boo
return nil return nil
} }
return []int{match.Index, match.Index + match.Length} return []int{match.Index, match.Index + match.Length}
// return []int{runePos(match.Index, string(str)), runePos(match.Index+match.Length, string(str))}
} }
func findAllIndex(regex *regexp.Regexp, str []byte, canMatchStart, canMatchEnd bool) [][]int { func findAllIndex(regex *regexp.Regexp, str []rune, canMatchStart, canMatchEnd bool) [][]int {
regexStr := regex.String() regexStr := regex.String()
if strings.Contains(regexStr, "^") { if strings.Contains(regexStr, "^") {
if !canMatchStart { if !canMatchStart {
@ -93,50 +94,56 @@ func findAllIndex(regex *regexp.Regexp, str []byte, canMatchStart, canMatchEnd b
return nil return nil
} }
} }
return regex.FindAllIndex([]byte(string(str)), -1) matches := regex.FindAllIndex([]byte(string(str)), -1)
for i, m := range matches {
matches[i][0] = runePos(m[0], string(str))
matches[i][1] = runePos(m[1], string(str))
}
return matches
} }
func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, region *Region, statesOnly bool) LineMatch { func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []rune, curRegion *region, statesOnly bool) LineMatch {
// highlights := make(LineMatch) // highlights := make(LineMatch)
if start == 0 { if start == 0 {
if !statesOnly { if !statesOnly {
highlights[0] = region.group highlights[0] = curRegion.group
} }
} }
loc := findIndex(region.end, line, start == 0, canMatchEnd) loc := findIndex(curRegion.end, line, start == 0, canMatchEnd)
if loc != nil { if loc != nil {
if !statesOnly { if !statesOnly {
highlights[start+runePos(loc[1]-1, string(line))] = region.group highlights[start+loc[1]-1] = curRegion.group
} }
if region.parent == nil { if curRegion.parent == nil {
if !statesOnly { if !statesOnly {
highlights[start+runePos(loc[1], string(line))] = 0 highlights[start+loc[1]] = 0
h.highlightRegion(highlights, start, false, lineNum, line[:loc[0]], region, statesOnly) h.highlightRegion(highlights, start, false, lineNum, line[:loc[0]], curRegion, statesOnly)
} }
h.highlightEmptyRegion(highlights, start+runePos(loc[1], string(line)), canMatchEnd, lineNum, line[loc[1]:], statesOnly) h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, line[loc[1]:], statesOnly)
return highlights return highlights
} }
if !statesOnly { if !statesOnly {
highlights[start+runePos(loc[1], string(line))] = region.parent.group highlights[start+loc[1]] = curRegion.parent.group
h.highlightRegion(highlights, start, false, lineNum, line[:loc[0]], region, statesOnly) h.highlightRegion(highlights, start, false, lineNum, line[:loc[0]], curRegion, statesOnly)
} }
h.highlightRegion(highlights, start+runePos(loc[1], string(line)), canMatchEnd, lineNum, line[loc[1]:], region.parent, statesOnly) h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, line[loc[1]:], curRegion.parent, statesOnly)
return highlights return highlights
} }
if len(line) == 0 || statesOnly { if len(line) == 0 || statesOnly {
if canMatchEnd { if canMatchEnd {
h.lastRegion = region h.lastRegion = curRegion
} }
return highlights return highlights
} }
firstLoc := []int{len(line), 0} firstLoc := []int{len(line), 0}
var firstRegion *Region
for _, r := range region.rules.regions { var firstRegion *region
for _, r := range curRegion.rules.regions {
loc := findIndex(r.start, line, start == 0, canMatchEnd) loc := findIndex(r.start, line, start == 0, canMatchEnd)
if loc != nil { if loc != nil {
if loc[0] < firstLoc[0] { if loc[0] < firstLoc[0] {
@ -146,18 +153,18 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE
} }
} }
if firstLoc[0] != len(line) { if firstLoc[0] != len(line) {
highlights[start+runePos(firstLoc[0], string(line))] = firstRegion.group highlights[start+firstLoc[0]] = firstRegion.group
h.highlightRegion(highlights, start, false, lineNum, line[:firstLoc[0]], region, statesOnly) h.highlightRegion(highlights, start, false, lineNum, line[:firstLoc[0]], curRegion, statesOnly)
h.highlightRegion(highlights, start+runePos(firstLoc[1], string(line)), canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion, statesOnly) h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion, statesOnly)
return highlights return highlights
} }
fullHighlights := make([]uint8, len(line)) fullHighlights := make([]Group, len([]rune(string(line))))
for i := 0; i < len(fullHighlights); i++ { for i := 0; i < len(fullHighlights); i++ {
fullHighlights[i] = region.group fullHighlights[i] = curRegion.group
} }
for _, p := range region.rules.patterns { for _, p := range curRegion.rules.patterns {
matches := findAllIndex(p.regex, line, start == 0, canMatchEnd) matches := findAllIndex(p.regex, line, start == 0, canMatchEnd)
for _, m := range matches { for _, m := range matches {
for i := m[0]; i < m[1]; i++ { for i := m[0]; i < m[1]; i++ {
@ -167,20 +174,20 @@ func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchE
} }
for i, h := range fullHighlights { for i, h := range fullHighlights {
if i == 0 || h != fullHighlights[i-1] { if i == 0 || h != fullHighlights[i-1] {
if _, ok := highlights[start+runePos(i, string(line))]; !ok { if _, ok := highlights[start+i]; !ok {
highlights[start+runePos(i, string(line))] = h highlights[start+i] = h
} }
} }
} }
if canMatchEnd { if canMatchEnd {
h.lastRegion = region h.lastRegion = curRegion
} }
return highlights return highlights
} }
func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch { func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []rune, statesOnly bool) LineMatch {
if len(line) == 0 { if len(line) == 0 {
if canMatchEnd { if canMatchEnd {
h.lastRegion = nil h.lastRegion = nil
@ -189,8 +196,8 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
} }
firstLoc := []int{len(line), 0} firstLoc := []int{len(line), 0}
var firstRegion *Region var firstRegion *region
for _, r := range h.def.rules.regions { for _, r := range h.Def.rules.regions {
loc := findIndex(r.start, line, start == 0, canMatchEnd) loc := findIndex(r.start, line, start == 0, canMatchEnd)
if loc != nil { if loc != nil {
if loc[0] < firstLoc[0] { if loc[0] < firstLoc[0] {
@ -201,10 +208,10 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
} }
if firstLoc[0] != len(line) { if firstLoc[0] != len(line) {
if !statesOnly { if !statesOnly {
highlights[start+runePos(firstLoc[0], string(line))] = firstRegion.group highlights[start+firstLoc[0]] = firstRegion.group
} }
h.highlightEmptyRegion(highlights, start, false, lineNum, line[:firstLoc[0]], statesOnly) h.highlightEmptyRegion(highlights, start, false, lineNum, line[:firstLoc[0]], statesOnly)
h.highlightRegion(highlights, start+runePos(firstLoc[1], string(line)), canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion, statesOnly) h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion, statesOnly)
return highlights return highlights
} }
@ -216,8 +223,8 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
return highlights return highlights
} }
fullHighlights := make([]uint8, len(line)) fullHighlights := make([]Group, len(line))
for _, p := range h.def.rules.patterns { for _, p := range h.Def.rules.patterns {
matches := findAllIndex(p.regex, line, start == 0, canMatchEnd) matches := findAllIndex(p.regex, line, start == 0, canMatchEnd)
for _, m := range matches { for _, m := range matches {
for i := m[0]; i < m[1]; i++ { for i := m[0]; i < m[1]; i++ {
@ -227,8 +234,8 @@ func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canM
} }
for i, h := range fullHighlights { for i, h := range fullHighlights {
if i == 0 || h != fullHighlights[i-1] { if i == 0 || h != fullHighlights[i-1] {
if _, ok := highlights[start+runePos(i, string(line))]; !ok { if _, ok := highlights[start+i]; !ok {
highlights[start+runePos(i, string(line))] = h highlights[start+i] = h
} }
} }
} }
@ -249,7 +256,7 @@ func (h *Highlighter) HighlightString(input string) []LineMatch {
var lineMatches []LineMatch var lineMatches []LineMatch
for i := 0; i < len(lines); i++ { for i := 0; i < len(lines); i++ {
line := []byte(lines[i]) line := []rune(lines[i])
highlights := make(LineMatch) highlights := make(LineMatch)
if i == 0 || h.lastRegion == nil { if i == 0 || h.lastRegion == nil {
@ -265,7 +272,7 @@ func (h *Highlighter) HighlightString(input string) []LineMatch {
// HighlightStates correctly sets all states for the buffer // HighlightStates correctly sets all states for the buffer
func (h *Highlighter) HighlightStates(input LineStates) { func (h *Highlighter) HighlightStates(input LineStates) {
for i := 0; i < input.LinesNum(); i++ { for i := 0; i < input.LinesNum(); i++ {
line := []byte(input.Line(i)) line := []rune(input.Line(i))
// highlights := make(LineMatch) // highlights := make(LineMatch)
if i == 0 || h.lastRegion == nil { if i == 0 || h.lastRegion == nil {
@ -289,7 +296,7 @@ func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int)
break break
} }
line := []byte(input.Line(i)) line := []rune(input.Line(i))
highlights := make(LineMatch) highlights := make(LineMatch)
var match LineMatch var match LineMatch
@ -313,7 +320,7 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) {
h.lastRegion = input.State(startline - 1) h.lastRegion = input.State(startline - 1)
} }
for i := startline; i < input.LinesNum(); i++ { for i := startline; i < input.LinesNum(); i++ {
line := []byte(input.Line(i)) line := []rune(input.Line(i))
// highlights := make(LineMatch) // highlights := make(LineMatch)
// var match LineMatch // var match LineMatch
@ -335,7 +342,7 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) {
// ReHighlightLine will rehighlight the state and match for a single line // ReHighlightLine will rehighlight the state and match for a single line
func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) {
line := []byte(input.Line(lineN)) line := []rune(input.Line(lineN))
highlights := make(LineMatch) highlights := make(LineMatch)
h.lastRegion = nil h.lastRegion = nil

58
cmd/micro/highlight/parser.go
View file

@ -9,12 +9,18 @@ import (
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
var Groups map[string]uint8 // A Group represents a syntax group
var numGroups uint8 type Group uint8
func GetGroup(n uint8) string { // Groups contains all of the groups that are defined
// You can access them in the map via their string name
var Groups map[string]Group
var numGroups Group
// String returns the group name attached to the specific group
func (g Group) String() string {
for k, v := range Groups { for k, v := range Groups {
if v == n { if v == g {
return k return k
} }
} }
@ -28,40 +34,40 @@ func GetGroup(n uint8) string {
type Def struct { type Def struct {
FileType string FileType string
ftdetect []*regexp.Regexp ftdetect []*regexp.Regexp
rules *Rules rules *rules
} }
// A Pattern is one simple syntax rule // A Pattern is one simple syntax rule
// It has a group that the rule belongs to, as well as // It has a group that the rule belongs to, as well as
// the regular expression to match the pattern // the regular expression to match the pattern
type Pattern struct { type pattern struct {
group uint8 group Group
regex *regexp.Regexp regex *regexp.Regexp
} }
// Rules defines which patterns and regions can be used to highlight // rules defines which patterns and regions can be used to highlight
// a filetype // a filetype
type Rules struct { type rules struct {
regions []*Region regions []*region
patterns []*Pattern patterns []*pattern
includes []string includes []string
} }
// A Region is a highlighted region (such as a multiline comment, or a string) // A region is a highlighted region (such as a multiline comment, or a string)
// It belongs to a group, and has start and end regular expressions // It belongs to a group, and has start and end regular expressions
// A Region also has rules of its own that only apply when matching inside the // A region also has rules of its own that only apply when matching inside the
// region and also rules from the above region do not match inside this region // region and also rules from the above region do not match inside this region
// Note that a region may contain more regions // Note that a region may contain more regions
type Region struct { type region struct {
group uint8 group Group
parent *Region parent *region
start *regexp2.Regexp start *regexp2.Regexp
end *regexp2.Regexp end *regexp2.Regexp
rules *Rules rules *rules
} }
func init() { func init() {
Groups = make(map[string]uint8) Groups = make(map[string]Group)
} }
// ParseDef parses an input syntax file into a highlight Def // ParseDef parses an input syntax file into a highlight Def
@ -118,6 +124,8 @@ func ParseDef(input []byte) (s *Def, err error) {
return s, err return s, err
} }
// ResolveIncludes will sort out the rules for including other filetypes
// You should call this after parsing all the Defs
func ResolveIncludes(defs []*Def) { func ResolveIncludes(defs []*Def) {
for _, d := range defs { for _, d := range defs {
resolveIncludesInDef(defs, d) resolveIncludesInDef(defs, d)
@ -139,7 +147,7 @@ func resolveIncludesInDef(defs []*Def, d *Def) {
} }
} }
func resolveIncludesInRegion(defs []*Def, region *Region) { func resolveIncludesInRegion(defs []*Def, region *region) {
for _, lang := range region.rules.includes { for _, lang := range region.rules.includes {
for _, searchDef := range defs { for _, searchDef := range defs {
if lang == searchDef.FileType { if lang == searchDef.FileType {
@ -154,8 +162,8 @@ func resolveIncludesInRegion(defs []*Def, region *Region) {
} }
} }
func parseRules(input []interface{}, curRegion *Region) (*Rules, error) { func parseRules(input []interface{}, curRegion *region) (*rules, error) {
rules := new(Rules) rules := new(rules)
for _, v := range input { for _, v := range input {
rule := v.(map[interface{}]interface{}) rule := v.(map[interface{}]interface{})
@ -179,10 +187,10 @@ func parseRules(input []interface{}, curRegion *Region) (*Rules, error) {
Groups[groupStr] = numGroups Groups[groupStr] = numGroups
} }
groupNum := Groups[groupStr] groupNum := Groups[groupStr]
rules.patterns = append(rules.patterns, &Pattern{groupNum, r}) rules.patterns = append(rules.patterns, &pattern{groupNum, r})
} }
case map[interface{}]interface{}: case map[interface{}]interface{}:
// Region // region
region, err := parseRegion(group.(string), object, curRegion) region, err := parseRegion(group.(string), object, curRegion)
if err != nil { if err != nil {
return nil, err return nil, err
@ -197,10 +205,10 @@ func parseRules(input []interface{}, curRegion *Region) (*Rules, error) {
return rules, nil return rules, nil
} }
func parseRegion(group string, regionInfo map[interface{}]interface{}, prevRegion *Region) (*Region, error) { func parseRegion(group string, regionInfo map[interface{}]interface{}, prevRegion *region) (*region, error) {
var err error var err error
region := new(Region) region := new(region)
if _, ok := Groups[group]; !ok { if _, ok := Groups[group]; !ok {
numGroups++ numGroups++
Groups[group] = numGroups Groups[group] = numGroups