add tests and patch problems

mdproc/comments.go

@@ -15,10 +15,15 @@ func RemoveComments(in chan pipe.StreamItem) chan pipe.StreamItem {
 
 		for b := range in {
 			data := b.Payload()
+			touched := false
 			for _, match := range re.FindAllSubmatch(data, -1) {
 				data = bytes.Replace(data, match[0], []byte(""), 1)
+				touched = true
 			}
-			out <- pipe.NewItem(b.Index(), append(bytes.TrimSpace(data), '\n'))
+			if touched && len(bytes.TrimSpace(data)) == 0 {
+				continue
+			}
+			out <- pipe.NewItem(b.Index(), data)
 		}
 
 		close(out)

mdproc/hugo.go

@@ -23,10 +23,9 @@ func RemoveFrontMatter(in chan pipe.StreamItem) chan pipe.StreamItem {
 				data = bytes.Replace(data, match[0], []byte(""), 1)
 				for _, title := range re2.FindAllSubmatch(match[0], 1) {
 					// add title
-					data = []byte(fmt.Sprintf("# %s\n\n", title[1]))
+					data = []byte(fmt.Sprintf("# %s\n", title[1]))
 				}
 			}
-			//out <- pipe.NewItem(b.Index(), append(bytes.TrimSpace(data), '\n'))
 			out <- pipe.NewItem(b.Index(), data)
 		}
 

mdproc/hugo_test.go

@@ -21,7 +21,7 @@ comment -->
 a quote
 
 This is
-a paragraph.
+a paragraph with a link to the [gemini protocol](https://en.wikipedia.org/wiki/Gemini_(protocol)).
 
 ` + "```" + `
 this is multi
@@ -30,8 +30,7 @@ line code
 
 and
 
-    this is code too
-`
+    this is code too`
 
 	preproc = `--- title: "This is the Title!" categories: [a,b] ---
 
@@ -39,7 +38,7 @@ and
 
 > this is a quote
 
-This is a paragraph.
+This is a paragraph with a link to the [gemini protocol](https://en.wikipedia.org/wiki/Gemini_(protocol)).
 
 ` + "```" + `
 this is multi
@@ -52,15 +51,15 @@ and
 this is code too
 ` + "```" + `
 
-
-
 `
 
 	gmi = `# This is the Title!
 
 > this is a quote
 
-This is a paragraph.
+This is a paragraph with a link to the gemini protocol[1].
+
+=> https://en.wikipedia.org/wiki/Gemini_(protocol) 1: gemini protocol
 
 ` + "```" + `
 this is multi
@@ -73,61 +72,45 @@ and
 this is code too
 ` + "```" + `
 
-
-
 `
 )
 
-func TestPreproc(t *testing.T) {
-	source := func() chan pipe.StreamItem {
-		data := make(chan pipe.StreamItem, len(strings.Split(input, "\n")))
-		for _, line := range strings.Split(input, "\n") {
+func source(in string) func() chan pipe.StreamItem {
+	return func() chan pipe.StreamItem {
+		data := make(chan pipe.StreamItem, len(strings.Split(in, "\n")))
+		for _, line := range strings.Split(in, "\n") {
 			data <- pipe.NewItem(0, []byte(line))
 		}
 		close(data)
 
 		return data
 	}
+}
 
-	sink := func(dest chan pipe.StreamItem) {
+func sink(t *testing.T, expected string) func(dest chan pipe.StreamItem) {
+	return func(dest chan pipe.StreamItem) {
 		var data []byte
 		for in := range dest {
 			data = append(data, in.Payload()...)
 		}
-		if string(data) != preproc {
-			t.Errorf("mismatch, expected '%s' but was '%s'", preproc, data)
+		if string(data) != expected {
+			t.Errorf("mismatch, expected '%s' but was '%s'", expected, data)
 		}
 	}
+}
 
-	sink(mdproc.Preproc()(source()))
+func TestPreproc(t *testing.T) {
+	s := pipe.New()
+	s.Use(mdproc.Preproc())
+	s.Handle(source(input), sink(t, preproc))
 }
 
 func TestMd2Gmi(t *testing.T) {
-	source := func() chan pipe.StreamItem {
-		data := make(chan pipe.StreamItem, len(strings.Split(input, "\n")))
-		for _, line := range strings.Split(input, "\n") {
-			data <- pipe.NewItem(0, []byte(line))
-		}
-		close(data)
-
-		return data
-	}
-
-	sink := func(dest chan pipe.StreamItem) {
-		var data []byte
-		for in := range dest {
-			data = append(data, in.Payload()...)
-		}
-		if string(data) != gmi {
-			t.Errorf("mismatch, expected '%s' but was '%s'", gmi, data)
-		}
-	}
-
 	s := pipe.New()
 	s.Use(mdproc.Preproc())
 	s.Use(mdproc.RemoveFrontMatter)
 	s.Use(mdproc.RemoveComments)
 	s.Use(mdproc.FormatHeadings)
 	s.Use(mdproc.FormatLinks)
-	s.Handle(source, sink)
+	s.Handle(source(input), sink(t, gmi))
 }

mdproc/links.go

@@ -38,7 +38,7 @@ func formatLinks(data []byte) []byte {
 	// find link name and url
 	var buffer []byte
 
-	re := regexp.MustCompile(`!?\[([^\]*]*)\]\(([^)]*)\)`)
+	re := regexp.MustCompile(`!?\[([^\]*]*)\]\(([^ ]*)\)`)
 
 	for i, match := range re.FindAllSubmatch(data, -1) {
 		replaceWithIndex := append(match[1], fmt.Sprintf("[%d]", i+1)...)
@@ -49,8 +49,8 @@ func formatLinks(data []byte) []byte {
 	}
 	// append links to that paragraph
 	if len(buffer) > 0 {
-		data = append(data, []byte("\n")...)
 		data = append(data, buffer...)
+		data = append(data, []byte("\n")...)
 	}
 
 	return data

mdproc/preproc.go

@@ -65,6 +65,7 @@ func (m *fsm) blockFlush() {
 	m.sendBuffer = append(m.sendBuffer, m.blockBuffer...)
 	m.blockBuffer = m.blockBuffer[:0]
 
+	// pending to sendbuffer too
 	if len(m.pending) > 0 {
 		m.sendBuffer = append(m.sendBuffer, m.pending...)
 		m.pending = m.pending[:0]
@@ -165,6 +166,7 @@ func paragraph(m *fsm, data []byte) stateFn {
 	if triggerBreak(data) {
 		m.blockBuffer = append(m.blockBuffer, data...)
 		m.blockBuffer = bytes.TrimSpace(m.blockBuffer)
+		// TODO, remove double spaces inside paragraphs
 		m.blockBuffer = append(m.blockBuffer, '\n')
 		m.blockFlush()