From 84e350aa6fe857d7ce934447d340d50f241d18d6 Mon Sep 17 00:00:00 2001 From: Zachary Yedidia Date: Tue, 2 May 2017 10:30:27 -0400 Subject: [PATCH] Optimize memory usage for loading syntax files --- cmd/micro/buffer.go | 66 ++++++++++++++++-- cmd/micro/cellview.go | 2 +- cmd/micro/highlight/ftdetect.go | 22 +++--- cmd/micro/highlight/parser.go | 119 ++++++++++++++++++++++---------- cmd/micro/highlighter.go | 8 +-- cmd/micro/micro.go | 8 +-- cmd/micro/settings.go | 6 +- 7 files changed, 163 insertions(+), 68 deletions(-) diff --git a/cmd/micro/buffer.go b/cmd/micro/buffer.go index 4b88f617..9fcf8a74 100644 --- a/cmd/micro/buffer.go +++ b/cmd/micro/buffer.go @@ -185,12 +185,66 @@ func (b *Buffer) GetName() string { // UpdateRules updates the syntax rules and filetype for this buffer // This is called when the colorscheme changes func (b *Buffer) UpdateRules() { - b.syntaxDef = highlight.DetectFiletype(syntaxDefs, b.Path, []byte(b.Line(0))) - if b.highlighter == nil || b.Settings["filetype"].(string) != b.syntaxDef.FileType { - b.Settings["filetype"] = b.syntaxDef.FileType - b.highlighter = highlight.NewHighlighter(b.syntaxDef) - if b.Settings["syntax"].(bool) { - b.highlighter.HighlightStates(b) + rehighlight := false + var files []*highlight.File + for _, f := range ListRuntimeFiles(RTSyntax) { + data, err := f.Data() + if err != nil { + TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error()) + } else { + file, err := highlight.ParseFile(data) + if err != nil { + TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error()) + continue + } + ftdetect, err := highlight.ParseFtDetect(file) + if err != nil { + TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error()) + continue + } + + ft := b.Settings["filetype"].(string) + if ft == "Unknown" || ft == "" { + if highlight.MatchFiletype(ftdetect, b.Path, b.lines[0].data) { + header := new(highlight.Header) + header.FileType = file.FileType + header.FtDetect = ftdetect + b.syntaxDef, err = highlight.ParseDef(file, header) + if err != nil { + TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error()) + continue + } + rehighlight = true + } + } else { + if file.FileType == ft { + header := new(highlight.Header) + header.FileType = file.FileType + header.FtDetect = ftdetect + b.syntaxDef, err = highlight.ParseDef(file, header) + if err != nil { + TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error()) + continue + } + rehighlight = true + } + } + files = append(files, file) + } + } + + if b.syntaxDef != nil { + highlight.ResolveIncludes(b.syntaxDef, files) + } + files = nil + + if b.highlighter == nil || rehighlight { + if b.syntaxDef != nil { + b.Settings["filetype"] = b.syntaxDef.FileType + b.highlighter = highlight.NewHighlighter(b.syntaxDef) + if b.Settings["syntax"].(bool) { + b.highlighter.HighlightStates(b) + } } } } diff --git a/cmd/micro/cellview.go b/cmd/micro/cellview.go index b864c6ba..2f0f8710 100644 --- a/cmd/micro/cellview.go +++ b/cmd/micro/cellview.go @@ -70,7 +70,7 @@ func (c *CellView) Draw(buf *Buffer, top, height, left, width int) { indentchar := []rune(buf.Settings["indentchar"].(string))[0] start := buf.Cursor.Y - if buf.Settings["syntax"].(bool) { + if buf.Settings["syntax"].(bool) && buf.syntaxDef != nil { if start > 0 && buf.lines[start-1].rehighlight { buf.highlighter.ReHighlightLine(buf, start-1) buf.lines[start-1].rehighlight = false diff --git a/cmd/micro/highlight/ftdetect.go b/cmd/micro/highlight/ftdetect.go index e04130fc..335295d5 100644 --- a/cmd/micro/highlight/ftdetect.go +++ b/cmd/micro/highlight/ftdetect.go @@ -1,22 +1,16 @@ package highlight +import "regexp" + // DetectFiletype will use the list of syntax definitions provided and the filename and first line of the file // to determine the filetype of the file // It will return the corresponding syntax definition for the filetype -func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def { - for _, d := range defs { - if d.ftdetect[0].MatchString(filename) { - return d - } - if len(d.ftdetect) > 1 { - if d.ftdetect[1].MatchString(string(firstLine)) { - return d - } - } +func MatchFiletype(ftdetect [2]*regexp.Regexp, filename string, firstLine []byte) bool { + return ftdetect[0].MatchString(filename) + + if ftdetect[1] != nil { + return ftdetect[1].Match(firstLine) } - emptyDef := new(Def) - emptyDef.FileType = "Unknown" - emptyDef.rules = new(rules) - return emptyDef + return false } diff --git a/cmd/micro/highlight/parser.go b/cmd/micro/highlight/parser.go index 6bf92704..47761608 100644 --- a/cmd/micro/highlight/parser.go +++ b/cmd/micro/highlight/parser.go @@ -30,9 +30,20 @@ func (g Group) String() string { // on filename or header (the first line of the file) // Then it has the rules which define how to highlight the file type Def struct { + *Header + + rules *rules +} + +type Header struct { FileType string - ftdetect []*regexp.Regexp - rules *rules + FtDetect [2]*regexp.Regexp +} + +type File struct { + FileType string + + yamlSrc map[interface{}]interface{} } // A Pattern is one simple syntax rule @@ -70,8 +81,41 @@ func init() { Groups = make(map[string]Group) } -// ParseDef parses an input syntax file into a highlight Def -func ParseDef(input []byte) (s *Def, err error) { +func ParseFtDetect(file *File) (r [2]*regexp.Regexp, err error) { + rules := file.yamlSrc + + loaded := 0 + for k, v := range rules { + if k == "detect" { + ftdetect := v.(map[interface{}]interface{}) + if len(ftdetect) >= 1 { + syntax, err := regexp.Compile(ftdetect["filename"].(string)) + if err != nil { + return r, err + } + + r[0] = syntax + } + if len(ftdetect) >= 2 { + header, err := regexp.Compile(ftdetect["header"].(string)) + if err != nil { + return r, err + } + + r[1] = header + } + loaded++ + } + + if loaded >= 2 { + break + } + } + + return r, err +} + +func ParseFile(input []byte) (f *File, err error) { // This is just so if we have an error, we can exit cleanly and return the parse error to the user defer func() { if e := recover(); e != nil { @@ -84,32 +128,37 @@ func ParseDef(input []byte) (s *Def, err error) { return nil, err } - s = new(Def) + f = new(File) + f.yamlSrc = rules for k, v := range rules { if k == "filetype" { filetype := v.(string) - s.FileType = filetype - } else if k == "detect" { - ftdetect := v.(map[interface{}]interface{}) - if len(ftdetect) >= 1 { - syntax, err := regexp.Compile(ftdetect["filename"].(string)) - if err != nil { - return nil, err - } + f.FileType = filetype + break + } + } - s.ftdetect = append(s.ftdetect, syntax) - } - if len(ftdetect) >= 2 { - header, err := regexp.Compile(ftdetect["header"].(string)) - if err != nil { - return nil, err - } + return f, err +} - s.ftdetect = append(s.ftdetect, header) - } - } else if k == "rules" { +// ParseDef parses an input syntax file into a highlight Def +func ParseDef(f *File, header *Header) (s *Def, err error) { + // This is just so if we have an error, we can exit cleanly and return the parse error to the user + defer func() { + if e := recover(); e != nil { + err = e.(error) + } + }() + + rules := f.yamlSrc + + s = new(Def) + s.Header = header + + for k, v := range rules { + if k == "rules" { inputRules := v.([]interface{}) rules, err := parseRules(inputRules, nil) @@ -126,38 +175,38 @@ func ParseDef(input []byte) (s *Def, err error) { // ResolveIncludes will sort out the rules for including other filetypes // You should call this after parsing all the Defs -func ResolveIncludes(defs []*Def) { - for _, d := range defs { - resolveIncludesInDef(defs, d) - } +func ResolveIncludes(def *Def, files []*File) { + resolveIncludesInDef(files, def) } -func resolveIncludesInDef(defs []*Def, d *Def) { +func resolveIncludesInDef(files []*File, d *Def) { for _, lang := range d.rules.includes { - for _, searchDef := range defs { - if lang == searchDef.FileType { + for _, searchFile := range files { + if lang == searchFile.FileType { + searchDef, _ := ParseDef(searchFile, nil) d.rules.patterns = append(d.rules.patterns, searchDef.rules.patterns...) d.rules.regions = append(d.rules.regions, searchDef.rules.regions...) } } } for _, r := range d.rules.regions { - resolveIncludesInRegion(defs, r) + resolveIncludesInRegion(files, r) r.parent = nil } } -func resolveIncludesInRegion(defs []*Def, region *region) { +func resolveIncludesInRegion(files []*File, region *region) { for _, lang := range region.rules.includes { - for _, searchDef := range defs { - if lang == searchDef.FileType { + for _, searchFile := range files { + if lang == searchFile.FileType { + searchDef, _ := ParseDef(searchFile, nil) region.rules.patterns = append(region.rules.patterns, searchDef.rules.patterns...) region.rules.regions = append(region.rules.regions, searchDef.rules.regions...) } } } for _, r := range region.rules.regions { - resolveIncludesInRegion(defs, r) + resolveIncludesInRegion(files, r) r.parent = region } } diff --git a/cmd/micro/highlighter.go b/cmd/micro/highlighter.go index ba88a705..53a46555 100644 --- a/cmd/micro/highlighter.go +++ b/cmd/micro/highlighter.go @@ -2,7 +2,7 @@ package main import "github.com/zyedidia/micro/cmd/micro/highlight" -var syntaxDefs []*highlight.Def +var syntaxFiles []*highlight.File func LoadSyntaxFiles() { InitColorscheme() @@ -14,17 +14,15 @@ func LoadSyntaxFiles() { LoadSyntaxFile(data, f.Name()) } } - - highlight.ResolveIncludes(syntaxDefs) } func LoadSyntaxFile(text []byte, filename string) { - def, err := highlight.ParseDef(text) + f, err := highlight.ParseFile(text) if err != nil { TermMessage("Syntax file error: " + filename + ": " + err.Error()) return } - syntaxDefs = append(syntaxDefs, def) + syntaxFiles = append(syntaxFiles, f) } diff --git a/cmd/micro/micro.go b/cmd/micro/micro.go index eb8622c6..49ae79d9 100644 --- a/cmd/micro/micro.go +++ b/cmd/micro/micro.go @@ -230,7 +230,7 @@ func LoadAll() { InitCommands() InitBindings() - LoadSyntaxFiles() + InitColorscheme() for _, tab := range tabs { for _, v := range tab.views { @@ -306,6 +306,7 @@ func main() { // This is used for sending the user messages in the bottom of the editor messenger = new(Messenger) messenger.history = make(map[string][]string) + InitColorscheme() // Now we load the input buffers := LoadInput() @@ -313,6 +314,7 @@ func main() { screen.Fini() os.Exit(1) } + for _, buf := range buffers { // For each buffer we create a new tab and place the view in that tab tab := NewTabFromView(NewView(buf)) @@ -384,12 +386,8 @@ func main() { LoadPlugins() - // Load the syntax files, including the colorscheme - LoadSyntaxFiles() - for _, t := range tabs { for _, v := range t.views { - v.Buf.UpdateRules() for pl := range loadedPlugins { _, err := Call(pl+".onViewOpen", v) if err != nil && !strings.HasPrefix(err.Error(), "function does not exist") { diff --git a/cmd/micro/settings.go b/cmd/micro/settings.go index 083b297f..560a1716 100644 --- a/cmd/micro/settings.go +++ b/cmd/micro/settings.go @@ -278,7 +278,8 @@ func SetOption(option, value string) error { globalSettings[option] = nativeValue if option == "colorscheme" { - LoadSyntaxFiles() + // LoadSyntaxFiles() + InitColorscheme() for _, tab := range tabs { for _, view := range tab.views { view.Buf.UpdateRules() @@ -342,7 +343,8 @@ func SetLocalOption(option, value string, view *View) error { } if option == "filetype" { - LoadSyntaxFiles() + // LoadSyntaxFiles() + InitColorscheme() buf.UpdateRules() }