Optimize memory usage for loading syntax files

cmd/micro/buffer.go

@@ -185,12 +185,66 @@ func (b *Buffer) GetName() string {
 // UpdateRules updates the syntax rules and filetype for this buffer
 // This is called when the colorscheme changes
 func (b *Buffer) UpdateRules() {
-	b.syntaxDef = highlight.DetectFiletype(syntaxDefs, b.Path, []byte(b.Line(0)))
-	if b.highlighter == nil || b.Settings["filetype"].(string) != b.syntaxDef.FileType {
-		b.Settings["filetype"] = b.syntaxDef.FileType
-		b.highlighter = highlight.NewHighlighter(b.syntaxDef)
-		if b.Settings["syntax"].(bool) {
-			b.highlighter.HighlightStates(b)
+	rehighlight := false
+	var files []*highlight.File
+	for _, f := range ListRuntimeFiles(RTSyntax) {
+		data, err := f.Data()
+		if err != nil {
+			TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+		} else {
+			file, err := highlight.ParseFile(data)
+			if err != nil {
+				TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+				continue
+			}
+			ftdetect, err := highlight.ParseFtDetect(file)
+			if err != nil {
+				TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+				continue
+			}
+
+			ft := b.Settings["filetype"].(string)
+			if ft == "Unknown" || ft == "" {
+				if highlight.MatchFiletype(ftdetect, b.Path, b.lines[0].data) {
+					header := new(highlight.Header)
+					header.FileType = file.FileType
+					header.FtDetect = ftdetect
+					b.syntaxDef, err = highlight.ParseDef(file, header)
+					if err != nil {
+						TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+						continue
+					}
+					rehighlight = true
+				}
+			} else {
+				if file.FileType == ft {
+					header := new(highlight.Header)
+					header.FileType = file.FileType
+					header.FtDetect = ftdetect
+					b.syntaxDef, err = highlight.ParseDef(file, header)
+					if err != nil {
+						TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+						continue
+					}
+					rehighlight = true
+				}
+			}
+			files = append(files, file)
+		}
+	}
+
+	if b.syntaxDef != nil {
+		highlight.ResolveIncludes(b.syntaxDef, files)
+	}
+	files = nil
+
+	if b.highlighter == nil || rehighlight {
+		if b.syntaxDef != nil {
+			b.Settings["filetype"] = b.syntaxDef.FileType
+			b.highlighter = highlight.NewHighlighter(b.syntaxDef)
+			if b.Settings["syntax"].(bool) {
+				b.highlighter.HighlightStates(b)
+			}
 		}
 	}
 }

cmd/micro/cellview.go

@@ -70,7 +70,7 @@ func (c *CellView) Draw(buf *Buffer, top, height, left, width int) {
 	indentchar := []rune(buf.Settings["indentchar"].(string))[0]
 
 	start := buf.Cursor.Y
-	if buf.Settings["syntax"].(bool) {
+	if buf.Settings["syntax"].(bool) && buf.syntaxDef != nil {
 		if start > 0 && buf.lines[start-1].rehighlight {
 			buf.highlighter.ReHighlightLine(buf, start-1)
 			buf.lines[start-1].rehighlight = false

cmd/micro/highlight/ftdetect.go

@@ -1,22 +1,16 @@
 package highlight
 
+import "regexp"
+
 // DetectFiletype will use the list of syntax definitions provided and the filename and first line of the file
 // to determine the filetype of the file
 // It will return the corresponding syntax definition for the filetype
-func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def {
-	for _, d := range defs {
-		if d.ftdetect[0].MatchString(filename) {
-			return d
-		}
-		if len(d.ftdetect) > 1 {
-			if d.ftdetect[1].MatchString(string(firstLine)) {
-				return d
-			}
-		}
+func MatchFiletype(ftdetect [2]*regexp.Regexp, filename string, firstLine []byte) bool {
+	return ftdetect[0].MatchString(filename)
+
+	if ftdetect[1] != nil {
+		return ftdetect[1].Match(firstLine)
 	}
 
-	emptyDef := new(Def)
-	emptyDef.FileType = "Unknown"
-	emptyDef.rules = new(rules)
-	return emptyDef
+	return false
 }

cmd/micro/highlight/parser.go

@@ -30,9 +30,20 @@ func (g Group) String() string {
 // on filename or header (the first line of the file)
 // Then it has the rules which define how to highlight the file
 type Def struct {
+	*Header
+
+	rules *rules
+}
+
+type Header struct {
 	FileType string
-	ftdetect []*regexp.Regexp
-	rules    *rules
+	FtDetect [2]*regexp.Regexp
+}
+
+type File struct {
+	FileType string
+
+	yamlSrc map[interface{}]interface{}
 }
 
 // A Pattern is one simple syntax rule
@@ -70,8 +81,41 @@ func init() {
 	Groups = make(map[string]Group)
 }
 
-// ParseDef parses an input syntax file into a highlight Def
-func ParseDef(input []byte) (s *Def, err error) {
+func ParseFtDetect(file *File) (r [2]*regexp.Regexp, err error) {
+	rules := file.yamlSrc
+
+	loaded := 0
+	for k, v := range rules {
+		if k == "detect" {
+			ftdetect := v.(map[interface{}]interface{})
+			if len(ftdetect) >= 1 {
+				syntax, err := regexp.Compile(ftdetect["filename"].(string))
+				if err != nil {
+					return r, err
+				}
+
+				r[0] = syntax
+			}
+			if len(ftdetect) >= 2 {
+				header, err := regexp.Compile(ftdetect["header"].(string))
+				if err != nil {
+					return r, err
+				}
+
+				r[1] = header
+			}
+			loaded++
+		}
+
+		if loaded >= 2 {
+			break
+		}
+	}
+
+	return r, err
+}
+
+func ParseFile(input []byte) (f *File, err error) {
 	// This is just so if we have an error, we can exit cleanly and return the parse error to the user
 	defer func() {
 		if e := recover(); e != nil {
@@ -84,32 +128,37 @@ func ParseDef(input []byte) (s *Def, err error) {
 		return nil, err
 	}
 
-	s = new(Def)
+	f = new(File)
+	f.yamlSrc = rules
 
 	for k, v := range rules {
 		if k == "filetype" {
 			filetype := v.(string)
 
-			s.FileType = filetype
-		} else if k == "detect" {
-			ftdetect := v.(map[interface{}]interface{})
-			if len(ftdetect) >= 1 {
-				syntax, err := regexp.Compile(ftdetect["filename"].(string))
-				if err != nil {
-					return nil, err
-				}
+			f.FileType = filetype
+			break
+		}
+	}
 
-				s.ftdetect = append(s.ftdetect, syntax)
-			}
-			if len(ftdetect) >= 2 {
-				header, err := regexp.Compile(ftdetect["header"].(string))
-				if err != nil {
-					return nil, err
-				}
+	return f, err
+}
 
-				s.ftdetect = append(s.ftdetect, header)
-			}
-		} else if k == "rules" {
+// ParseDef parses an input syntax file into a highlight Def
+func ParseDef(f *File, header *Header) (s *Def, err error) {
+	// This is just so if we have an error, we can exit cleanly and return the parse error to the user
+	defer func() {
+		if e := recover(); e != nil {
+			err = e.(error)
+		}
+	}()
+
+	rules := f.yamlSrc
+
+	s = new(Def)
+	s.Header = header
+
+	for k, v := range rules {
+		if k == "rules" {
 			inputRules := v.([]interface{})
 
 			rules, err := parseRules(inputRules, nil)
@@ -126,38 +175,38 @@ func ParseDef(input []byte) (s *Def, err error) {
 
 // ResolveIncludes will sort out the rules for including other filetypes
 // You should call this after parsing all the Defs
-func ResolveIncludes(defs []*Def) {
-	for _, d := range defs {
-		resolveIncludesInDef(defs, d)
-	}
+func ResolveIncludes(def *Def, files []*File) {
+	resolveIncludesInDef(files, def)
 }
 
-func resolveIncludesInDef(defs []*Def, d *Def) {
+func resolveIncludesInDef(files []*File, d *Def) {
 	for _, lang := range d.rules.includes {
-		for _, searchDef := range defs {
-			if lang == searchDef.FileType {
+		for _, searchFile := range files {
+			if lang == searchFile.FileType {
+				searchDef, _ := ParseDef(searchFile, nil)
 				d.rules.patterns = append(d.rules.patterns, searchDef.rules.patterns...)
 				d.rules.regions = append(d.rules.regions, searchDef.rules.regions...)
 			}
 		}
 	}
 	for _, r := range d.rules.regions {
-		resolveIncludesInRegion(defs, r)
+		resolveIncludesInRegion(files, r)
 		r.parent = nil
 	}
 }
 
-func resolveIncludesInRegion(defs []*Def, region *region) {
+func resolveIncludesInRegion(files []*File, region *region) {
 	for _, lang := range region.rules.includes {
-		for _, searchDef := range defs {
-			if lang == searchDef.FileType {
+		for _, searchFile := range files {
+			if lang == searchFile.FileType {
+				searchDef, _ := ParseDef(searchFile, nil)
 				region.rules.patterns = append(region.rules.patterns, searchDef.rules.patterns...)
 				region.rules.regions = append(region.rules.regions, searchDef.rules.regions...)
 			}
 		}
 	}
 	for _, r := range region.rules.regions {
-		resolveIncludesInRegion(defs, r)
+		resolveIncludesInRegion(files, r)
 		r.parent = region
 	}
 }

cmd/micro/highlighter.go

@@ -2,7 +2,7 @@ package main
 
 import "github.com/zyedidia/micro/cmd/micro/highlight"
 
-var syntaxDefs []*highlight.Def
+var syntaxFiles []*highlight.File
 
 func LoadSyntaxFiles() {
 	InitColorscheme()
@@ -14,17 +14,15 @@ func LoadSyntaxFiles() {
 			LoadSyntaxFile(data, f.Name())
 		}
 	}
-
-	highlight.ResolveIncludes(syntaxDefs)
 }
 
 func LoadSyntaxFile(text []byte, filename string) {
-	def, err := highlight.ParseDef(text)
+	f, err := highlight.ParseFile(text)
 
 	if err != nil {
 		TermMessage("Syntax file error: " + filename + ": " + err.Error())
 		return
 	}
 
-	syntaxDefs = append(syntaxDefs, def)
+	syntaxFiles = append(syntaxFiles, f)
 }

cmd/micro/micro.go

@@ -230,7 +230,7 @@ func LoadAll() {
 	InitCommands()
 	InitBindings()
 
-	LoadSyntaxFiles()
+	InitColorscheme()
 
 	for _, tab := range tabs {
 		for _, v := range tab.views {
@@ -306,6 +306,7 @@ func main() {
 	// This is used for sending the user messages in the bottom of the editor
 	messenger = new(Messenger)
 	messenger.history = make(map[string][]string)
+	InitColorscheme()
 
 	// Now we load the input
 	buffers := LoadInput()
@@ -313,6 +314,7 @@ func main() {
 		screen.Fini()
 		os.Exit(1)
 	}
+
 	for _, buf := range buffers {
 		// For each buffer we create a new tab and place the view in that tab
 		tab := NewTabFromView(NewView(buf))
@@ -384,12 +386,8 @@ func main() {
 
 	LoadPlugins()
 
-	// Load the syntax files, including the colorscheme
-	LoadSyntaxFiles()
-
 	for _, t := range tabs {
 		for _, v := range t.views {
-			v.Buf.UpdateRules()
 			for pl := range loadedPlugins {
 				_, err := Call(pl+".onViewOpen", v)
 				if err != nil && !strings.HasPrefix(err.Error(), "function does not exist") {

cmd/micro/settings.go

@@ -278,7 +278,8 @@ func SetOption(option, value string) error {
 	globalSettings[option] = nativeValue
 
 	if option == "colorscheme" {
-		LoadSyntaxFiles()
+		// LoadSyntaxFiles()
+		InitColorscheme()
 		for _, tab := range tabs {
 			for _, view := range tab.views {
 				view.Buf.UpdateRules()
@@ -342,7 +343,8 @@ func SetLocalOption(option, value string, view *View) error {
 	}
 
 	if option == "filetype" {
-		LoadSyntaxFiles()
+		// LoadSyntaxFiles()
+		InitColorscheme()
 		buf.UpdateRules()
 	}