From 0b920bd80a5dcd594858356ed774859a913b3ef2 Mon Sep 17 00:00:00 2001 From: Steve Gargan Date: Wed, 17 Apr 2019 21:48:47 +0100 Subject: [PATCH] Modify whitespace trimming to be less hungry The liquid spec is a little vague about quite how greedy whitespace trimming should be. Currently handling here will eat any available whitespace between tags including all newline characters e.g. ``` ( {%- sometag -%} ) ``` will render as ``` () ``` This is ok for web markup where the whitespace is not important, but for other document types this greedy consumption is problematic. Ideally the ws trimming would be less hungry and only trim any whitespace on the line containing the tag up to and including it's newline, essentially removing any trace of the tag but leaving other lines intact rendering the above as ``` ( ) ``` The change here detects whitespace as tokens as they are parsed and discards any as indicated by the trimming directives up to an including the first newline. --- parser/parser.go | 30 +++++++++ parser/parser_test.go | 34 +++++++++- parser/scanner.go | 29 +++++---- parser/scanner_test.go | 113 ++++++++++++++++++++++----------- parser/token.go | 8 ++- parser/tokentype_string.go | 8 +-- render/nodes.go | 2 +- render/render.go | 28 +++----- render/render_test.go | 4 +- render/trimwriter.go | 62 ------------------ tags/control_flow_tags_test.go | 50 +++++++++++++++ tags/iteration_tags_test.go | 35 +++++++++- tags/standard_tags_test.go | 22 +++++++ 13 files changed, 283 insertions(+), 142 deletions(-) delete mode 100644 render/trimwriter.go diff --git a/parser/parser.go b/parser/parser.go index 9211a15d..edf3dc27 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -22,6 +22,10 @@ func (c Config) parseTokens(tokens []Token) (ASTNode, Error) { // nolint: gocycl node *ASTBlock ap *[]ASTNode } + type WhitespaceControl struct { + stack []ASTNode // stack to buffer ws for trimming + trimRight bool + } var ( g = c.Grammar root = &ASTSeq{} // root of AST; will be returned @@ -32,8 +36,18 @@ func (c Config) parseTokens(tokens []Token) (ASTNode, Error) { // nolint: gocycl rawTag *ASTRaw // current raw tag inComment = false inRaw = false + wsc = &WhitespaceControl{} ) for _, tok := range tokens { + // check the current tag for trim left in the current tag if true + // then drop any stacked whitespace. + if tok.Type != WhitespaceTokenType { + if !tok.TrimLeft { + *ap = append(*ap, wsc.stack...) + } + wsc.stack = wsc.stack[:0] + wsc.trimRight = tok.TrimRight + } switch { // The parser needs to know about comment and raw, because tags inside // needn't match each other e.g. {%comment%}{%if%}{%endcomment%} @@ -56,6 +70,19 @@ func (c Config) parseTokens(tokens []Token) (ASTNode, Error) { // nolint: gocycl *ap = append(*ap, &ASTObject{tok, expr}) case tok.Type == TextTokenType: *ap = append(*ap, &ASTText{Token: tok}) + case tok.Type == WhitespaceTokenType: + // append to the ws stack unless the previous node requested + // ws should be trimmed + if !wsc.trimRight { + wsc.stack = append(wsc.stack, &ASTText{Token: tok}) + } + if tok.Name == "New Line" { + // trimming should only occur up to the first newline + // so it is safe to append the stack now + *ap = append(*ap, wsc.stack...) + wsc.stack = wsc.stack[:0] + wsc.trimRight = false + } case tok.Type == TagTokenType: if cs, ok := g.BlockSyntax(tok.Name); ok { switch { @@ -101,5 +128,8 @@ func (c Config) parseTokens(tokens []Token) (ASTNode, Error) { // nolint: gocycl if bn != nil { return nil, Errorf(bn, "unterminated %q block", bn.Name) } + + // append any whitespace still queued + *ap = append(*ap, wsc.stack...) return root, nil } diff --git a/parser/parser_test.go b/parser/parser_test.go index 78393f70..be7252b8 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -43,7 +43,6 @@ var parserTests = []struct{ in string }{ {`{% unless test %}{% endunless %}`}, {`{% for item in list %}{% if test %}{% else %}{% endif %}{% endfor %}`}, {`{% if true %}{% raw %}{% endraw %}{% endif %}`}, - {`{% comment %}{% if true %}{% endcomment %}`}, {`{% raw %}{% if true %}{% endraw %}`}, } @@ -68,3 +67,36 @@ func TestParser(t *testing.T) { }) } } + +var parseWhitespaceTests = []struct { + in string + expected int +}{ + // expected counts include object tokens + {"{{ obj -}} \t\n\t {{ obj }}", 3}, + {"{{ obj }} \t\n\t {{- obj }}", 4}, + {"{{ obj -}} \t\n\t {{- obj }}", 2}, + {"{{ obj -}} \t\n\t\n\t {{- obj }}", 4}, // preseves mid whitespace + + // expected counts for whitespace in clause do not include if tags + {"{% if test -%} \t\n\t {% endif %}", 1}, + {"{% if test %} \t\n\t {%- endif %}", 2}, + {"{% if test -%} \t\n\t {%- endif %}", 0}, + {"{% if test -%} \t\n\t\n\t {%- endif %}", 2}, +} + +func TestParseWhitespace(t *testing.T) { + cfg := Config{Grammar: grammarFake{}} + for i, test := range parseWhitespaceTests { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + ast, _ := cfg.Parse(test.in, SourceLoc{}) + children := ast.(*ASTSeq).Children + switch children[0].(type) { + case *ASTSeq: + require.Equal(t, len(children), test.expected) + case *ASTBlock: + require.Equal(t, len(children[0].(*ASTBlock).Body), test.expected) + } + }) + } +} diff --git a/parser/scanner.go b/parser/scanner.go index a63b97e4..ef9e82be 100644 --- a/parser/scanner.go +++ b/parser/scanner.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strings" + "unicode" ) // Scan breaks a string into a sequence of Tokens. @@ -20,12 +21,18 @@ func Scan(data string, loc SourceLoc, delims []string) (tokens []Token) { p, pe := 0, len(data) for _, m := range tokenMatcher.FindAllStringSubmatchIndex(data, -1) { ts, te := m[0], m[1] + source := data[ts:te] if p < ts { tokens = append(tokens, Token{Type: TextTokenType, SourceLoc: loc, Source: data[p:ts]}) - loc.LineNo += strings.Count(data[p:ts], "\n") } - source := data[ts:te] switch { + case rune(data[ts]) == '\n': + tok := Token{Type: WhitespaceTokenType, Name: "New Line", SourceLoc: loc, Source: source} + loc.LineNo++ + tokens = append(tokens, tok) + case unicode.IsSpace(rune(data[ts])): + tok := Token{Type: WhitespaceTokenType, Name: "Whitespace", SourceLoc: loc, Source: source} + tokens = append(tokens, tok) case data[ts:ts+len(delims[0])] == delims[0]: tok := Token{ Type: ObjTokenType, @@ -41,16 +48,15 @@ func Scan(data string, loc SourceLoc, delims []string) (tokens []Token) { Type: TagTokenType, SourceLoc: loc, Source: source, - Name: data[m[4]:m[5]], + Name: data[m[8]:m[9]], TrimLeft: source[2] == '-', TrimRight: source[len(source)-3] == '-', } - if m[6] > 0 { - tok.Args = data[m[6]:m[7]] + if m[10] > 0 { + tok.Args = data[m[10]:m[11]] } tokens = append(tokens, tok) } - loc.LineNo += strings.Count(source, "\n") p = te } if p < pe { @@ -73,13 +79,12 @@ func formTokenMatcher(delims []string) *regexp.Regexp { } } - tokenMatcher := regexp.MustCompile( - fmt.Sprintf(`%s-?\s*(.+?)\s*-?%s|%s-?\s*(\w+)(?:\s+((?:%v)+?))?\s*-?%s`, - // QuoteMeta will escape any of these that are regex commands - regexp.QuoteMeta(delims[0]), regexp.QuoteMeta(delims[1]), - regexp.QuoteMeta(delims[2]), strings.Join(exclusion, "|"), regexp.QuoteMeta(delims[3]), - ), + p := fmt.Sprintf(`%s-?\s*(.+?)\s*-?%s|([ \t]+)|(\n)|%s-?\s*(\w+)(?:\s+((?:%v)+?))?\s*-?%s`, + // QuoteMeta will escape any of these that are regex commands + regexp.QuoteMeta(delims[0]), regexp.QuoteMeta(delims[1]), + regexp.QuoteMeta(delims[2]), strings.Join(exclusion, "|"), regexp.QuoteMeta(delims[3]), ) + tokenMatcher := regexp.MustCompile(p) return tokenMatcher } diff --git a/parser/scanner_test.go b/parser/scanner_test.go index ef52c133..48381567 100644 --- a/parser/scanner_test.go +++ b/parser/scanner_test.go @@ -25,39 +25,29 @@ var scannerCountTests = []struct { func TestScan(t *testing.T) { scan := func(src string) []Token { return Scan(src, SourceLoc{}, nil) } tokens := scan("12") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TextTokenType, tokens[0].Type) + verifyTokens(t, TextTokenType, 1, tokens) require.Equal(t, "12", tokens[0].Source) tokens = scan("{{obj}}") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, ObjTokenType, tokens[0].Type) + verifyTokens(t, ObjTokenType, 1, tokens) require.Equal(t, "obj", tokens[0].Args) tokens = scan("{{ obj }}") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, ObjTokenType, tokens[0].Type) + verifyTokens(t, ObjTokenType, 1, tokens) require.Equal(t, "obj", tokens[0].Args) tokens = scan("{%tag args%}") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TagTokenType, tokens[0].Type) + verifyTokens(t, TagTokenType, 1, tokens) require.Equal(t, "tag", tokens[0].Name) require.Equal(t, "args", tokens[0].Args) tokens = scan("{% tag args %}") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TagTokenType, tokens[0].Type) + verifyTokens(t, TagTokenType, 1, tokens) require.Equal(t, "tag", tokens[0].Name) require.Equal(t, "args", tokens[0].Args) tokens = scan("pre{% tag args %}mid{{ object }}post") - require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args"} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens)) + require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args", l: false, r: false} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens)) for i, test := range scannerCountTests { t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { @@ -78,14 +68,11 @@ func TestScan_ws(t *testing.T) { {`{{ expr }}`, "expr", false, false}, {`{{- expr }}`, "expr", true, false}, {`{{ expr -}}`, "expr", false, true}, - {`{% tag arg %}`, "tag", false, false}, - {`{%- tag arg %}`, "tag", true, false}, - {`{% tag arg -%}`, "tag", false, true}, + {`{{- expr -}}`, "expr", true, true}, } for i, test := range wsTests { t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { tokens := scan(test.in) - require.Len(t, tokens, 1) tok := tokens[0] if test.expect == "tag" { require.Equalf(t, "tag", tok.Name, test.in) @@ -99,6 +86,61 @@ func TestScan_ws(t *testing.T) { } } +func TestScanWhiteSpaceTokens(t *testing.T) { + // whitespace control + scan := func(src string) []Token { return Scan(src, SourceLoc{}, nil) } + + wsTests := []struct { + in string + numTokens int + expected []TokenType + }{ + {" ", 1, []TokenType{WhitespaceTokenType}}, + {" ", 1, []TokenType{WhitespaceTokenType}}, + {"\n", 1, []TokenType{WhitespaceTokenType}}, + {"\t", 1, []TokenType{WhitespaceTokenType}}, + {"\t\t\t\t", 1, []TokenType{WhitespaceTokenType}}, + {"\t\n\t", 3, []TokenType{WhitespaceTokenType, WhitespaceTokenType, WhitespaceTokenType}}, + {"{{ expr }} {{ expr }}", 3, []TokenType{ObjTokenType, WhitespaceTokenType, ObjTokenType}}, + {"{{ expr }}\t\n\t{{ expr }}", 5, []TokenType{ObjTokenType, WhitespaceTokenType, WhitespaceTokenType, WhitespaceTokenType, ObjTokenType}}, + {"{{ expr }}\t \t\n\t \t{{ expr }}", 5, []TokenType{ObjTokenType, WhitespaceTokenType, WhitespaceTokenType, WhitespaceTokenType, ObjTokenType}}, + {"{{ expr }}\t \t\nSomeText\n\t \t{{ expr }}", 7, []TokenType{ObjTokenType, WhitespaceTokenType, WhitespaceTokenType, TextTokenType, WhitespaceTokenType, WhitespaceTokenType, ObjTokenType}}, + } + for i, test := range wsTests { + t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { + tokens := scan(test.in) + require.Len(t, tokens, test.numTokens) + for x, tok := range tokens { + require.Equal(t, test.expected[x], tok.Type) + } + }) + } +} + +func TestScanTokenLocationParsing(t *testing.T) { + // whitespace control + scan := func(src string) []Token { return Scan(src, SourceLoc{LineNo: 1}, nil) } + + wsTests := []struct { + in string + expectedLineNos []int + }{ + {"\t \t \tsometext", []int{1, 1}}, + {"\t\n\t", []int{1, 1, 2}}, + {"\nsometext", []int{1, 2}}, + {"{{ expr }}\t \t\nSomeText\n\t \t{{ expr }}", []int{1, 1, 1, 2, 2, 3, 3}}, + } + for i, test := range wsTests { + t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { + tokens := scan(test.in) + require.Len(t, tokens, len(test.expectedLineNos)) + for x, tok := range tokens { + require.Equal(t, test.expectedLineNos[x], tok.SourceLoc.LineNo) + } + }) + } +} + var scannerCountTestsDelims = []struct { in string len int @@ -119,39 +161,29 @@ func TestScan_delims(t *testing.T) { return Scan(src, SourceLoc{}, []string{"OBJECT@LEFT", "OBJECT#RIGHT", "TAG*LEFT", "TAG!RIGHT"}) } tokens := scan("12") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TextTokenType, tokens[0].Type) + verifyTokens(t, TextTokenType, 1, tokens) require.Equal(t, "12", tokens[0].Source) tokens = scan("OBJECT@LEFTobjOBJECT#RIGHT") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, ObjTokenType, tokens[0].Type) + verifyTokens(t, ObjTokenType, 1, tokens) require.Equal(t, "obj", tokens[0].Args) tokens = scan("OBJECT@LEFT obj OBJECT#RIGHT") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, ObjTokenType, tokens[0].Type) + verifyTokens(t, ObjTokenType, 1, tokens) require.Equal(t, "obj", tokens[0].Args) tokens = scan("TAG*LEFTtag argsTAG!RIGHT") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TagTokenType, tokens[0].Type) + verifyTokens(t, TagTokenType, 1, tokens) require.Equal(t, "tag", tokens[0].Name) require.Equal(t, "args", tokens[0].Args) tokens = scan("TAG*LEFT tag args TAG!RIGHT") - require.NotNil(t, tokens) - require.Len(t, tokens, 1) - require.Equal(t, TagTokenType, tokens[0].Type) + verifyTokens(t, TagTokenType, 1, tokens) require.Equal(t, "tag", tokens[0].Name) require.Equal(t, "args", tokens[0].Args) - tokens = scan("preTAG*LEFT tag args TAG!RIGHTmidOBJECT@LEFT object OBJECT#RIGHTpost") - require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args"} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens)) + tokens = scan("\npreTAG*LEFT tag args TAG!RIGHTmidOBJECT@LEFT object OBJECT#RIGHTpost\t") + require.Equal(t, `[WhitespaceTokenType{"New Line"} TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args", l: false, r: false} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"} WhitespaceTokenType{"Whitespace"}]`, fmt.Sprint(tokens)) for i, test := range scannerCountTestsDelims { t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { @@ -160,3 +192,10 @@ func TestScan_delims(t *testing.T) { }) } } + +func verifyTokens(t require.TestingT, tokenType TokenType, length int, tokens []Token) []Token { + require.NotNil(t, tokens) + require.Len(t, tokens, length) + require.Equal(t, tokenType, tokens[0].Type) + return tokens +} diff --git a/parser/token.go b/parser/token.go index 08df029e..7c7c613e 100644 --- a/parser/token.go +++ b/parser/token.go @@ -15,7 +15,7 @@ type Token struct { // TokenType is the type of a Chunk type TokenType int -////go:generate stringer -type=TokenType +//go:generate stringer -type=TokenType const ( // TextTokenType is the type of a text Chunk @@ -24,6 +24,8 @@ const ( TagTokenType // ObjTokenType is the type of an object Chunk "{{…}}" ObjTokenType + // WhitespaceTokenType represents whitespace + WhitespaceTokenType ) // SourceLoc contains a Token's source location. @@ -48,9 +50,11 @@ func (c Token) String() string { case TextTokenType: return fmt.Sprintf("%v{%#v}", c.Type, c.Source) case TagTokenType: - return fmt.Sprintf("%v{Tag:%#v, Args:%#v}", c.Type, c.Name, c.Args) + return fmt.Sprintf("%v{Tag:%#v, Args:%#v, l: %#v, r: %#v}", c.Type, c.Name, c.Args, c.TrimLeft, c.TrimRight) case ObjTokenType: return fmt.Sprintf("%v{%#v}", c.Type, c.Args) + case WhitespaceTokenType: + return fmt.Sprintf("%v{%#v}", c.Type, c.Name) default: return fmt.Sprintf("%v{%#v}", c.Type, c.Source) } diff --git a/parser/tokentype_string.go b/parser/tokentype_string.go index 6230c80f..1d5f3875 100644 --- a/parser/tokentype_string.go +++ b/parser/tokentype_string.go @@ -2,15 +2,15 @@ package parser -import "fmt" +import "strconv" -const _TokenType_name = "TextTokenTypeTagTokenTypeObjTokenType" +const _TokenType_name = "TextTokenTypeTagTokenTypeObjTokenTypeWhitespaceTokenType" -var _TokenType_index = [...]uint8{0, 13, 25, 37} +var _TokenType_index = [...]uint8{0, 13, 25, 37, 56} func (i TokenType) String() string { if i < 0 || i >= TokenType(len(_TokenType_index)-1) { - return fmt.Sprintf("TokenType(%d)", i) + return "TokenType(" + strconv.FormatInt(int64(i), 10) + ")" } return _TokenType_name[_TokenType_index[i]:_TokenType_index[i+1]] } diff --git a/render/nodes.go b/render/nodes.go index 6695d2fa..bf66e294 100644 --- a/render/nodes.go +++ b/render/nodes.go @@ -11,7 +11,7 @@ import ( type Node interface { SourceLocation() parser.SourceLoc // for error reporting SourceText() string // for error reporting - render(*trimWriter, nodeContext) Error + render(io.Writer, nodeContext) Error } // BlockNode represents a {% tag %}…{% endtag %}. diff --git a/render/render.go b/render/render.go index 3e8b8129..9b33776b 100644 --- a/render/render.go +++ b/render/render.go @@ -12,31 +12,23 @@ import ( // Render renders the render tree. func Render(node Node, w io.Writer, vars map[string]interface{}, c Config) Error { - tw := trimWriter{w: w} - if err := node.render(&tw, newNodeContext(vars, c)); err != nil { + if err := node.render(w, newNodeContext(vars, c)); err != nil { return err } - if err := tw.Flush(); err != nil { - panic(err) - } return nil } // RenderASTSequence renders a sequence of nodes. func (c nodeContext) RenderSequence(w io.Writer, seq []Node) Error { - tw := trimWriter{w: w} for _, n := range seq { - if err := n.render(&tw, c); err != nil { + if err := n.render(w, c); err != nil { return err } } - if err := tw.Flush(); err != nil { - panic(err) - } return nil } -func (n *BlockNode) render(w *trimWriter, ctx nodeContext) Error { +func (n *BlockNode) render(w io.Writer, ctx nodeContext) Error { cd, ok := ctx.config.findBlockDef(n.Name) if !ok || cd.parser == nil { // this should have been detected during compilation; it's an implementation error if it happens here @@ -50,7 +42,7 @@ func (n *BlockNode) render(w *trimWriter, ctx nodeContext) Error { return wrapRenderError(err, n) } -func (n *RawNode) render(w *trimWriter, ctx nodeContext) Error { +func (n *RawNode) render(w io.Writer, ctx nodeContext) Error { for _, s := range n.slices { _, err := io.WriteString(w, s) if err != nil { @@ -60,8 +52,7 @@ func (n *RawNode) render(w *trimWriter, ctx nodeContext) Error { return nil } -func (n *ObjectNode) render(w *trimWriter, ctx nodeContext) Error { - w.TrimLeft(n.TrimLeft) +func (n *ObjectNode) render(w io.Writer, ctx nodeContext) Error { value, err := ctx.Evaluate(n.expr) if err != nil { return wrapRenderError(err, n) @@ -69,11 +60,10 @@ func (n *ObjectNode) render(w *trimWriter, ctx nodeContext) Error { if err := wrapRenderError(writeObject(w, value), n); err != nil { return err } - w.TrimRight(n.TrimRight) return nil } -func (n *SeqNode) render(w *trimWriter, ctx nodeContext) Error { +func (n *SeqNode) render(w io.Writer, ctx nodeContext) Error { for _, c := range n.Children { if err := c.render(w, ctx); err != nil { return err @@ -82,14 +72,12 @@ func (n *SeqNode) render(w *trimWriter, ctx nodeContext) Error { return nil } -func (n *TagNode) render(w *trimWriter, ctx nodeContext) Error { - w.TrimLeft(n.TrimLeft) +func (n *TagNode) render(w io.Writer, ctx nodeContext) Error { err := wrapRenderError(n.renderer(w, rendererContext{ctx, n, nil}), n) - w.TrimRight(n.TrimRight) return err } -func (n *TextNode) render(w *trimWriter, ctx nodeContext) Error { +func (n *TextNode) render(w io.Writer, ctx nodeContext) Error { _, err := io.WriteString(w, n.Source) return wrapRenderError(err, n) } diff --git a/render/render_test.go b/render/render_test.go index db8d334a..683dc184 100644 --- a/render/render_test.go +++ b/render/render_test.go @@ -29,7 +29,7 @@ var renderTests = []struct{ in, out string }{ {`{{ array[1] }}`, "second"}, // whitespace control - {` {{ 1 }} `, " 1 "}, + {` {{ 1 }} `, " 1 "}, {` {{- 1 }} `, "1 "}, {` {{ 1 -}} `, " 1"}, {` {{- 1 -}} `, "1"}, @@ -92,7 +92,7 @@ func TestRender(t *testing.T) { buf := new(bytes.Buffer) err = Render(root, buf, renderTestBindings, cfg) require.NoErrorf(t, err, test.in) - require.Equalf(t, test.out, buf.String(), test.in) + require.Equalf(t, test.out, buf.String(), "[%v]", test.in) }) } } diff --git a/render/trimwriter.go b/render/trimwriter.go deleted file mode 100644 index c3f9894f..00000000 --- a/render/trimwriter.go +++ /dev/null @@ -1,62 +0,0 @@ -package render - -import ( - "bytes" - "io" - "unicode" -) - -// A trimWriter provides whitespace control around a wrapped io.Writer. -// The caller should call TrimLeft(bool) and TrimRight(bool) respectively -// before and after processing a tag or expression, and Flush() at completion. -type trimWriter struct { - w io.Writer - buf bytes.Buffer - trimRight bool -} - -// This violates the letter of the protocol by returning the count of the -// bytes, rather than the actual number of bytes written. We can't know the -// number of bytes written until later, and it won't in general be the same -// as the argument length (that's the whole point of trimming), but speaking -// truthfully here would cause some callers to return io.ErrShortWrite, ruining -// this as an io.Writer. -func (tw *trimWriter) Write(b []byte) (int, error) { - n := len(b) - if tw.trimRight { - b = bytes.TrimLeftFunc(b, unicode.IsSpace) - } else if tw.buf.Len() > 0 { - if err := tw.Flush(); err != nil { - return 0, err - } - } - nonWS := bytes.TrimRightFunc(b, unicode.IsSpace) - if len(nonWS) < len(b) { - if _, err := tw.buf.Write(b[len(nonWS):]); err != nil { - return 0, err - } - } - _, err := tw.w.Write(nonWS) - return n, err -} -func (tw *trimWriter) Flush() (err error) { - if tw.buf.Len() > 0 { - _, err = tw.buf.WriteTo(tw.w) - tw.buf.Reset() - } - return -} - -func (tw *trimWriter) TrimLeft(f bool) { - if !f && tw.buf.Len() > 0 { - if err := tw.Flush(); err != nil { - panic(err) - } - } - tw.buf.Reset() - tw.trimRight = false -} - -func (tw *trimWriter) TrimRight(f bool) { - tw.trimRight = f -} diff --git a/tags/control_flow_tags_test.go b/tags/control_flow_tags_test.go index 8e357167..84046cc5 100644 --- a/tags/control_flow_tags_test.go +++ b/tags/control_flow_tags_test.go @@ -58,6 +58,41 @@ var cfTagErrorTests = []struct{ in, expected string }{ {`{% case a | undefined_filter %}{% when 1 %}{% endcase %}`, "undefined filter"}, } +var cfTagWhitespaceTests = []struct{ in, expected string }{ + {` {%- if true %} trims outside {% endif -%} `, " trims outside "}, + {` ({% if true -%} trims inside {%- endif %}) `, " (trims inside) "}, + {`( {%- if true -%} trims both {%- endif -%} )`, "(trims both)"}, + {`removes +{%- if true -%} +block +{%- endif -%} +lines`, + `removes +block +lines`}, + {`removes +{%- if true -%} +block +{%- else -%} +not rendered +{%- endif -%} +lines`, + `removes +block +lines`}, + {`removes +{%- case 1 -%} +{%- when 1 -%} +block +{%- when 2 -%} +not rendered +{%- endcase -%} +lines`, + `removes +block +lines`}, +} + func TestControlFlowTags(t *testing.T) { cfg := render.NewConfig() AddStandardTags(cfg) @@ -73,6 +108,21 @@ func TestControlFlowTags(t *testing.T) { } } +func TestControlFlowTagsWithWhitespace(t *testing.T) { + cfg := render.NewConfig() + AddStandardTags(cfg) + for i, test := range cfTagWhitespaceTests { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + root, err := cfg.Compile(test.in, parser.SourceLoc{}) + require.NoErrorf(t, err, test.in) + buf := new(bytes.Buffer) + err = render.Render(root, buf, tagTestBindings, cfg) + require.NoErrorf(t, err, test.in) + require.Equalf(t, test.expected, buf.String(), test.in) + }) + } +} + func TestControlFlowTags_errors(t *testing.T) { cfg := render.NewConfig() AddStandardTags(cfg) diff --git a/tags/iteration_tags_test.go b/tags/iteration_tags_test.go index d3acfecc..dcdb08c1 100644 --- a/tags/iteration_tags_test.go +++ b/tags/iteration_tags_test.go @@ -92,7 +92,40 @@ var iterationTests = []struct{ in, expected string }{ {`{% tablerow product in products cols:2 %}{{ product }}{% endtablerow %}`, `Cool ShirtAlien Poster Batman PosterBullseye Shirt - Another Classic VinylAwesome Jeans`}, + Another Classic VinylAwesome Jeans`}, + + {`before +{% for a in array %}{{ a }} +{%- endfor -%} +after`, `before +first +second +third +after`}, + + {`before +{% for a in array %} +{{ a }} +{% endfor %} +after`, `before + +first + +second + +third + +after`}, + + {`before +{%- for a in array -%} +{{ a }} +{%- endfor -%} +after`, `before +first +second +third +after`}, } var iterationSyntaxErrorTests = []struct{ in, expected string }{ diff --git a/tags/standard_tags_test.go b/tags/standard_tags_test.go index a0cbf6c6..77c9bb8e 100644 --- a/tags/standard_tags_test.go +++ b/tags/standard_tags_test.go @@ -34,6 +34,13 @@ var tagTests = []struct{ in, expected string }{ {`pre{% raw %}{% if false %}anyway-{% endraw %}post`, "pre{% if false %}anyway-post"}, } +var tagWhitespaceTests = []struct{ in, expected string }{ + // variable tags + {" {%- assign av = 1 -%}\n({{- av -}} )", "(1)"}, + {"( {%- capture x -%} \t\ncaptured\t {%- endcapture %}{{ x -}} )", "(captured)"}, + {"( {%- comment %}\n{{ a }}\n{% undefined_tag %}{% endcomment -%} )", "()"}, +} + var tagErrorTests = []struct{ in, expected string }{ {`{% assign av = x | undefined_filter %}`, "undefined filter"}, } @@ -93,6 +100,21 @@ func TestStandardTags(t *testing.T) { } } +func TestStandardTagsWithWhitespace(t *testing.T) { + config := render.NewConfig() + AddStandardTags(config) + for i, test := range tagWhitespaceTests { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + root, err := config.Compile(test.in, parser.SourceLoc{}) + require.NoErrorf(t, err, test.in) + buf := new(bytes.Buffer) + err = render.Render(root, buf, tagTestBindings, config) + require.NoErrorf(t, err, test.in) + require.Equalf(t, test.expected, buf.String(), test.in) + }) + } +} + func TestStandardTags_render_errors(t *testing.T) { config := render.NewConfig() AddStandardTags(config)