add to tests and improve parser

mdproc/hugo.go

@@ -22,11 +22,12 @@ func RemoveFrontMatter(in chan pipe.StreamItem) chan pipe.StreamItem {
 			for _, match := range re.FindAllSubmatch(data, -1) {
 				data = bytes.Replace(data, match[0], []byte(""), 1)
 				for _, title := range re2.FindAllSubmatch(match[0], 1) {
-					// prefix title
+					// add title
-					data = append([]byte(fmt.Sprintf("# %s\n\n", title[1])), data...)
+					data = []byte(fmt.Sprintf("# %s\n\n", title[1]))
 				}
 			}
-			out <- pipe.NewItem(b.Index(), append(bytes.TrimSpace(data), '\n'))
+			//out <- pipe.NewItem(b.Index(), append(bytes.TrimSpace(data), '\n'))
+			out <- pipe.NewItem(b.Index(), data)
 		}
 
 		close(out)

mdproc/hugo_test.go

@@ -8,11 +8,8 @@ import (
 	"github.com/n0x1m/md2gmi/pipe"
 )
 
-func TestHugoMarkdownToGmi(t *testing.T) {
+const (
-
+	input = `---
-	source := func() chan pipe.StreamItem {
-		input := `
----
 title: "This is the Title!"
 categories: [a,b]
 ---
@@ -27,10 +24,62 @@ This is
 a paragraph.
 
 ` + "```" + `
-this is
+this is multi
-code
+line code
 ` + "```" + `
+
+and
+
+    this is code too
 `
+
+	preproc = `--- title: "This is the Title!" categories: [a,b] ---
+
+<!-- a comment -->
+
+> this is a quote
+
+This is a paragraph.
+
+` + "```" + `
+this is multi
+line code
+` + "```" + `
+
+and
+
+` + "```" + `
+this is code too
+` + "```" + `
+
+
+
+`
+
+	gmi = `# This is the Title!
+
+> this is a quote
+
+This is a paragraph.
+
+` + "```" + `
+this is multi
+line code
+` + "```" + `
+
+and
+
+` + "```" + `
+this is code too
+` + "```" + `
+
+
+
+`
+)
+
+func TestPreproc(t *testing.T) {
+	source := func() chan pipe.StreamItem {
 		data := make(chan pipe.StreamItem, len(strings.Split(input, "\n")))
 		for _, line := range strings.Split(input, "\n") {
 			data <- pipe.NewItem(0, []byte(line))
@@ -45,37 +94,40 @@ code
 		for in := range dest {
 			data = append(data, in.Payload()...)
 		}
-		expected := `
+		if string(data) != preproc {
-
+			t.Errorf("mismatch, expected '%s' but was '%s'", preproc, data)
---- title: "This is the Title!" categories: [a,b] ---
-
-<!-- a comment -->
-
-> this is a quote
-
-This is a paragraph.
-
-
-
-` + "```" + `
-this is
-code
-` + "```" + `
-
-
-
-`
-		if string(data) != expected {
-			t.Errorf("mismatch, expected '%s' but was '%s'", expected, data)
 		}
 	}
 
 	sink(mdproc.Preproc()(source()))
-	//s := pipe.New()
+}
-	//s.Use(mdproc.Preproc())
+
-	//s.Use(mdproc.RemoveFrontMatter)
+func TestMd2Gmi(t *testing.T) {
-	//s.Use(mdproc.RemoveComments)
+	source := func() chan pipe.StreamItem {
-	//s.Use(mdproc.FormatHeadings)
+		data := make(chan pipe.StreamItem, len(strings.Split(input, "\n")))
-	//s.Use(mdproc.FormatLinks)
+		for _, line := range strings.Split(input, "\n") {
-	//s.Handle(source, sink)
+			data <- pipe.NewItem(0, []byte(line))
+		}
+		close(data)
+
+		return data
+	}
+
+	sink := func(dest chan pipe.StreamItem) {
+		var data []byte
+		for in := range dest {
+			data = append(data, in.Payload()...)
+		}
+		if string(data) != gmi {
+			t.Errorf("mismatch, expected '%s' but was '%s'", gmi, data)
+		}
+	}
+
+	s := pipe.New()
+	s.Use(mdproc.Preproc())
+	s.Use(mdproc.RemoveFrontMatter)
+	s.Use(mdproc.RemoveComments)
+	s.Use(mdproc.FormatHeadings)
+	s.Use(mdproc.FormatLinks)
+	s.Handle(source, sink)
 }

mdproc/preproc.go

@@ -67,7 +67,6 @@ func (m *fsm) blockFlush() {
 
 	if len(m.pending) > 0 {
 		m.sendBuffer = append(m.sendBuffer, m.pending...)
-		m.sendBuffer = append(m.sendBuffer, '\n')
 		m.pending = m.pending[:0]
 	}
 }
@@ -100,6 +99,9 @@ func needsFence(data []byte) bool {
 }
 
 func normal(m *fsm, data []byte) stateFn {
+	if len(data) == 0 {
+		return normal
+	}
 	if data, isList := handleList(data); isList {
 		m.blockBuffer = append(data, '\n')
 		m.blockFlush()