From bb42042db421c7835ab026b5c8c9ec55df8cd4e1 Mon Sep 17 00:00:00 2001 From: Steven Date: Sat, 16 Dec 2023 12:48:52 +0800 Subject: [PATCH] chore: implement task list parser --- plugin/gomark/ast/block.go | 13 +++++ plugin/gomark/ast/node.go | 1 + plugin/gomark/parser/parser.go | 9 ++-- plugin/gomark/parser/parser_test.go | 54 +++++++++++++++++++ plugin/gomark/parser/task_list.go | 69 ++++++++++++++++++++++++ plugin/gomark/parser/task_list_test.go | 57 ++++++++++++++++++++ plugin/gomark/renderer/html/html.go | 20 +++++++ plugin/gomark/renderer/html/html_test.go | 8 +++ plugin/gomark/renderer/string/string.go | 8 +++ 9 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 plugin/gomark/parser/task_list.go create mode 100644 plugin/gomark/parser/task_list_test.go diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go index c69d6a6a9..2b3ec4c7e 100644 --- a/plugin/gomark/ast/block.go +++ b/plugin/gomark/ast/block.go @@ -87,3 +87,16 @@ type UnorderedList struct { func (*UnorderedList) Type() NodeType { return UnorderedListNode } + +type TaskList struct { + BaseBlock + + // Symbol is "*" or "-" or "+". + Symbol string + Complete bool + Children []Node +} + +func (*TaskList) Type() NodeType { + return TaskListNode +} diff --git a/plugin/gomark/ast/node.go b/plugin/gomark/ast/node.go index 6552555e5..59a4162e7 100644 --- a/plugin/gomark/ast/node.go +++ b/plugin/gomark/ast/node.go @@ -13,6 +13,7 @@ const ( BlockquoteNode OrderedListNode UnorderedListNode + TaskListNode // Inline nodes. TextNode BoldNode diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.go index 9c647f36e..3ceadcb2a 100644 --- a/plugin/gomark/parser/parser.go +++ b/plugin/gomark/parser/parser.go @@ -25,21 +25,22 @@ type BlockParser interface { BaseParser } +func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) { + return ParseBlock(tokens) +} + var defaultBlockParsers = []BlockParser{ NewCodeBlockParser(), NewHorizontalRuleParser(), NewHeadingParser(), NewBlockquoteParser(), + NewTaskListParser(), NewUnorderedListParser(), NewOrderedListParser(), NewParagraphParser(), NewLineBreakParser(), } -func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) { - return ParseBlock(tokens) -} - func ParseBlock(tokens []*tokenizer.Token) ([]ast.Node, error) { return ParseBlockWithParsers(tokens, defaultBlockParsers) } diff --git a/plugin/gomark/parser/parser_test.go b/plugin/gomark/parser/parser_test.go index b1c489e58..cbde6b948 100644 --- a/plugin/gomark/parser/parser_test.go +++ b/plugin/gomark/parser/parser_test.go @@ -1,6 +1,7 @@ package parser import ( + "strconv" "testing" "github.com/stretchr/testify/require" @@ -151,6 +152,51 @@ func TestParser(t *testing.T) { }, }, }, + { + text: "1. hello\n- [ ] world", + nodes: []ast.Node{ + &ast.OrderedList{ + Number: "1", + Children: []ast.Node{ + &ast.Text{ + Content: "hello", + }, + }, + }, + &ast.TaskList{ + Symbol: tokenizer.Hyphen, + Complete: false, + Children: []ast.Node{ + &ast.Text{ + Content: "world", + }, + }, + }, + }, + }, + { + text: "- [ ] hello\n- [x] world", + nodes: []ast.Node{ + &ast.TaskList{ + Symbol: tokenizer.Hyphen, + Complete: false, + Children: []ast.Node{ + &ast.Text{ + Content: "hello", + }, + }, + }, + &ast.TaskList{ + Symbol: tokenizer.Hyphen, + Complete: true, + Children: []ast.Node{ + &ast.Text{ + Content: "world", + }, + }, + }, + }, + }, } for _, test := range tests { @@ -184,6 +230,12 @@ func StringifyNode(node ast.Node) string { return "HorizontalRule(" + n.Symbol + ")" case *ast.Blockquote: return "Blockquote(" + StringifyNodes(n.Children) + ")" + case *ast.OrderedList: + return "OrderedList(" + n.Number + ", " + StringifyNodes(n.Children) + ")" + case *ast.UnorderedList: + return "UnorderedList(" + n.Symbol + ", " + StringifyNodes(n.Children) + ")" + case *ast.TaskList: + return "TaskList(" + n.Symbol + ", " + strconv.FormatBool(n.Complete) + ", " + StringifyNodes(n.Children) + ")" case *ast.Text: return "Text(" + n.Content + ")" case *ast.Bold: @@ -202,6 +254,8 @@ func StringifyNode(node ast.Node) string { return "Tag(" + n.Content + ")" case *ast.Strikethrough: return "Strikethrough(" + n.Content + ")" + case *ast.EscapingCharacter: + return "EscapingCharacter(" + n.Symbol + ")" } return "" } diff --git a/plugin/gomark/parser/task_list.go b/plugin/gomark/parser/task_list.go new file mode 100644 index 000000000..abb62399a --- /dev/null +++ b/plugin/gomark/parser/task_list.go @@ -0,0 +1,69 @@ +package parser + +import ( + "errors" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +type TaskListParser struct{} + +func NewTaskListParser() *TaskListParser { + return &TaskListParser{} +} + +func (*TaskListParser) Match(tokens []*tokenizer.Token) (int, bool) { + if len(tokens) < 7 { + return 0, false + } + + symbolToken := tokens[0] + if symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign { + return 0, false + } + if tokens[1].Type != tokenizer.Space { + return 0, false + } + if tokens[2].Type != tokenizer.LeftSquareBracket || (tokens[3].Type != tokenizer.Space && tokens[3].Value != "x") || tokens[4].Type != tokenizer.RightSquareBracket { + return 0, false + } + if tokens[5].Type != tokenizer.Space { + return 0, false + } + + contentTokens := []*tokenizer.Token{} + for _, token := range tokens[6:] { + contentTokens = append(contentTokens, token) + if token.Type == tokenizer.Newline { + break + } + } + if len(contentTokens) == 0 { + return 0, false + } + + return len(contentTokens) + 6, true +} + +func (p *TaskListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) { + size, ok := p.Match(tokens) + if size == 0 || !ok { + return nil, errors.New("not matched") + } + + symbolToken := tokens[0] + contentTokens := tokens[6:size] + if contentTokens[len(contentTokens)-1].Type == tokenizer.Newline { + contentTokens = contentTokens[:len(contentTokens)-1] + } + children, err := ParseInline(contentTokens) + if err != nil { + return nil, err + } + return &ast.TaskList{ + Symbol: symbolToken.Type, + Complete: tokens[3].Value == "x", + Children: children, + }, nil +} diff --git a/plugin/gomark/parser/task_list_test.go b/plugin/gomark/parser/task_list_test.go new file mode 100644 index 000000000..3676264b2 --- /dev/null +++ b/plugin/gomark/parser/task_list_test.go @@ -0,0 +1,57 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/plugin/gomark/ast" + "github.com/usememos/memos/plugin/gomark/parser/tokenizer" +) + +func TestTaskListParser(t *testing.T) { + tests := []struct { + text string + node ast.Node + }{ + { + text: "*asd", + node: nil, + }, + { + text: "+ [ ] Hello World", + node: &ast.TaskList{ + Symbol: tokenizer.PlusSign, + Complete: false, + Children: []ast.Node{ + &ast.Text{ + Content: "Hello World", + }, + }, + }, + }, + { + text: "* [x] **Hello**", + node: &ast.TaskList{ + Symbol: tokenizer.Asterisk, + Complete: true, + Children: []ast.Node{ + &ast.Bold{ + Symbol: "*", + Children: []ast.Node{ + &ast.Text{ + Content: "Hello", + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + tokens := tokenizer.Tokenize(test.text) + node, _ := NewTaskListParser().Parse(tokens) + require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node})) + } +} diff --git a/plugin/gomark/renderer/html/html.go b/plugin/gomark/renderer/html/html.go index 1c6a4d4e6..5422f2864 100644 --- a/plugin/gomark/renderer/html/html.go +++ b/plugin/gomark/renderer/html/html.go @@ -43,6 +43,8 @@ func (r *HTMLRenderer) RenderNode(node ast.Node) { r.renderUnorderedList(n) case *ast.OrderedList: r.renderOrderedList(n) + case *ast.TaskList: + r.renderTaskList(n) case *ast.Bold: r.renderBold(n) case *ast.Italic: @@ -119,6 +121,24 @@ func (r *HTMLRenderer) renderBlockquote(node *ast.Blockquote) { } } +func (r *HTMLRenderer) renderTaskList(node *ast.TaskList) { + prevSibling, nextSibling := node.PrevSibling(), node.NextSibling() + if prevSibling == nil || prevSibling.Type() != ast.TaskListNode { + r.output.WriteString("") + } +} + func (r *HTMLRenderer) renderUnorderedList(node *ast.UnorderedList) { prevSibling, nextSibling := node.PrevSibling(), node.NextSibling() if prevSibling == nil || prevSibling.Type() != ast.UnorderedListNode { diff --git a/plugin/gomark/renderer/html/html_test.go b/plugin/gomark/renderer/html/html_test.go index 90db196cc..4df788204 100644 --- a/plugin/gomark/renderer/html/html_test.go +++ b/plugin/gomark/renderer/html/html_test.go @@ -54,6 +54,14 @@ func TestHTMLRenderer(t *testing.T) { text: "1. Hello\n2. world\n* !", expected: `
  1. Hello
  2. world
`, }, + { + text: "- [ ] hello\n- [x] world", + expected: ``, + }, + { + text: "1. ordered\n* unorder\n- [ ] checkbox\n- [x] checked", + expected: `
  1. ordered
`, + }, } for _, test := range tests { diff --git a/plugin/gomark/renderer/string/string.go b/plugin/gomark/renderer/string/string.go index bf125d18a..8e80a088c 100644 --- a/plugin/gomark/renderer/string/string.go +++ b/plugin/gomark/renderer/string/string.go @@ -39,6 +39,8 @@ func (r *StringRenderer) RenderNode(node ast.Node) { r.renderHorizontalRule(n) case *ast.Blockquote: r.renderBlockquote(n) + case *ast.TaskList: + r.renderTaskList(n) case *ast.UnorderedList: r.renderUnorderedList(n) case *ast.OrderedList: @@ -109,6 +111,12 @@ func (r *StringRenderer) renderBlockquote(node *ast.Blockquote) { r.output.WriteString("\n") } +func (r *StringRenderer) renderTaskList(node *ast.TaskList) { + r.output.WriteString(node.Symbol) + r.RenderNodes(node.Children) + r.output.WriteString("\n") +} + func (r *StringRenderer) renderUnorderedList(node *ast.UnorderedList) { r.output.WriteString(node.Symbol) r.RenderNodes(node.Children)