more fixes for multi line, nested comments

mdproc/preproc.go

@@ -61,8 +61,8 @@ func wrap(m *fsm, data []byte) (*fsm, []byte) {
 		m.multiLineBlockMode -= ecount
 	}
 
-	// clip entire line
+	// clip entire line if no control sequences present
-	if m.multiLineBlockMode > 0 {
+	if (m.multiLineBlockMode > 0 && scount == 0 && ecount == 0) || m.multiLineBlockMode > 1 {
 		data = data[:0]
 		return m, data
 	}
@@ -149,8 +149,8 @@ func handleList(data []byte) ([]byte, bool) {
 	return data, false
 }
 
-func isFence(data []byte) bool {
+func hasFence(data []byte) bool {
-	return len(data) >= 3 && string(data[0:3]) == "```"
+	return bytes.Contains(data, []byte("```"))
 }
 
 func needsFence(data []byte) bool {
@@ -169,7 +169,7 @@ func normalText(m *fsm, data []byte) stateFn {
 		return list
 	}
 
-	if isFence(data) {
+	if hasFence(data) {
 		m.blockBuffer = append(data, '\n')
 
 		return fence
@@ -215,7 +215,7 @@ func list(m *fsm, data []byte) stateFn {
 func fence(m *fsm, data []byte) stateFn {
 	m.blockBuffer = append(m.blockBuffer, append(data, '\n')...)
 	// second fence returns to normal
-	if isFence(data) {
+	if hasFence(data) {
 		m.softBlockFlush()
 
 		return normal

mdproc/preproc_test.go

@@ -0,0 +1,157 @@
+package mdproc_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/n0x1m/md2gmi/mdproc"
+	"github.com/n0x1m/md2gmi/pipe"
+)
+
+const (
+	input = `---
+title: "This is the Title!"
+categories: [a,b]
+---
+
+<!-- a <!-- double -->
+comment -->
+
+<!--
+multi line comment block, terminated.
+
+With a break line, <!-- and multi line
+ inline comments -->
+and [with a](link);
+-->
+> this is
+a quote
+
+*not a list*<!-- comment
+between two things -->
+**also not a list**
+
+ - this
+	- is
+* an unordered
+  * list
+
+This is
+a paragraph with a link to the [gemini protocol](https://en.wikipedia.org/wiki/Gemini_(protocol)).
+
+` + "```" + `
+this is multi
+line code
+` + "```" + `
+
+and<!-- inline comment-->
+
+    this is code too`
+
+	preproc = `--- title: "This is the Title!" categories: [a,b] ---
+
+> this is a quote
+
+*not a list*
+
+**also not a list**
+
+- this
+- is
+- an unordered
+- list
+
+This is a paragraph with a link to the [gemini protocol](https://en.wikipedia.org/wiki/Gemini_(protocol)).
+
+` + "```" + `
+this is multi
+line code
+` + "```" + `
+
+and
+
+` + "```" + `
+this is code too
+` + "```" + `
+
+`
+
+	gmi = `# This is the Title!
+
+> this is a quote
+
+*not a list*
+
+**also not a list**
+
+- this
+- is
+- an unordered
+- list
+
+This is a paragraph with a link to the gemini protocol[1].
+
+=> https://en.wikipedia.org/wiki/Gemini_(protocol) 1: gemini protocol
+
+` + "```" + `
+this is multi
+line code
+` + "```" + `
+
+and
+
+` + "```" + `
+this is code too
+` + "```" + `
+
+`
+)
+
+func source(t *testing.T, in string) func() chan pipe.StreamItem {
+	t.Helper()
+
+	return func() chan pipe.StreamItem {
+		data := make(chan pipe.StreamItem, len(strings.Split(in, "\n")))
+		for _, line := range strings.Split(in, "\n") {
+			data <- pipe.NewItem(0, []byte(line))
+		}
+
+		close(data)
+
+		return data
+	}
+}
+
+func sink(t *testing.T, expected string) func(dest chan pipe.StreamItem) {
+	t.Helper()
+
+	return func(dest chan pipe.StreamItem) {
+		var data []byte
+		for in := range dest {
+			data = append(data, in.Payload()...)
+		}
+
+		if string(data) != expected {
+			t.Errorf("mismatch, expected '%s' but was '%s'", expected, data)
+		}
+	}
+}
+
+func TestPreproc(t *testing.T) {
+	t.Parallel()
+
+	s := pipe.New()
+	s.Use(mdproc.Preprocessor())
+	s.Handle(source(t, input), sink(t, preproc))
+}
+
+func TestMd2Gmi(t *testing.T) {
+	t.Parallel()
+
+	s := pipe.New()
+	s.Use(mdproc.Preprocessor())
+	s.Use(mdproc.RemoveFrontMatter)
+	s.Use(mdproc.FormatHeadings)
+	s.Use(mdproc.FormatLinks)
+	s.Handle(source(t, input), sink(t, gmi))
+}