Merge pull request #38 from pleask/master

Allow multiline string literal constants

Author: yoheimuta

.gitignore

@@ -39,3 +39,5 @@ atlassian-ide-plugin.xml
 # Cursive Clojure plugin
 .idea/replstate.xml
 
+.config/
+tags/

internal/lexer/constant.go

@@ -1,17 +1,24 @@
 package lexer
 
-import "github.com/yoheimuta/go-protoparser/internal/lexer/scanner"
+import (
+	"strings"
 
-// ReadConstant reads a constant.
+	"github.com/yoheimuta/go-protoparser/internal/lexer/scanner"
+)
+
+// ReadConstant reads a constant. If permissive is true, accepts multiline string literals.
 // constant = fullIdent | ( [ "-" | "+" ] intLit ) | ( [ "-" | "+" ] floatLit ) | strLit | boolLit
-func (lex *Lexer) ReadConstant() (string, scanner.Position, error) {
+func (lex *Lexer) ReadConstant(permissive bool) (string, scanner.Position, error) {
 	lex.NextLit()
 
 	startPos := lex.Pos
 	cons := lex.Text
 
 	switch {
 	case lex.Token == scanner.TSTRLIT:
+		if permissive {
+			return lex.mergeMultilineStrLit(), startPos, nil
+		}
 		return cons, startPos, nil
 	case lex.Token == scanner.TBOOLLIT:
 		return cons, startPos, nil
@@ -38,3 +45,21 @@ func (lex *Lexer) ReadConstant() (string, scanner.Position, error) {
 		return "", scanner.Position{}, lex.unexpected(lex.Text, "constant")
 	}
 }
+
+// Merges a multiline string literal into a single string.
+func (lex *Lexer) mergeMultilineStrLit() string {
+	q := "'"
+	if strings.HasPrefix(lex.Text, "\"") {
+		q = "\""
+	}
+	var b strings.Builder
+	b.WriteString(q)
+	for lex.Token == scanner.TSTRLIT {
+		strippedString := strings.Trim(lex.Text, q)
+		b.WriteString(strippedString)
+		lex.NextLit()
+	}
+	lex.UnNext()
+	b.WriteString(q)
+	return b.String()
+}

internal/lexer/constant_test.go

@@ -58,11 +58,23 @@ func TestLexer2_ReadConstant(t *testing.T) {
 			wantIsEOF: true,
 		},
 		{
-			name:      "strLit",
+			name:      "single line strLit",
 			input:     `"あいうえお''"`,
 			wantText:  `"あいうえお''"`,
 			wantIsEOF: true,
 		},
+		{
+			name:      "multiline strLit with double quotes",
+			input:     "\"line1 \"\n\"line2 \" \n\"line3\" ",
+			wantText:  `"line1 line2 line3"`,
+			wantIsEOF: true,
+		},
+		{
+			name:      "multiline strLit with single quotes",
+			input:     "'line1 '\n'line2 ' \n'line3' ",
+			wantText:  `'line1 line2 line3'`,
+			wantIsEOF: true,
+		},
 		{
 			name:      "boolLit",
 			input:     "true",
@@ -89,7 +101,7 @@ func TestLexer2_ReadConstant(t *testing.T) {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
 			lex := lexer.NewLexer(strings.NewReader(test.input))
-			got, pos, err := lex.ReadConstant()
+			got, pos, err := lex.ReadConstant(true)
 
 			switch {
 			case test.wantErr:

parser/enum.go

@@ -2,7 +2,6 @@ package parser
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/yoheimuta/go-protoparser/internal/lexer/scanner"
 	"github.com/yoheimuta/go-protoparser/parser/meta"
@@ -309,52 +308,13 @@ func (p *Parser) parseEnumValueOption() (*EnumValueOption, error) {
 		return nil, p.unexpected("=")
 	}
 
-	constant, _, err := p.lex.ReadConstant()
+	constant, _, err := p.lex.ReadConstant(p.permissive)
 	if err != nil {
 		return nil, err
 	}
 
-	if p.permissive {
-		// accept a multiple string literals. See https://github.com/yoheimuta/go-protoparser/issues/35.
-		for {
-			next := p.lex.Peek()
-			if next == scanner.TCOMMA || next == scanner.TRIGHTSQUARE {
-				break
-			}
-
-			lit, _, err := p.lex.ReadConstant()
-			if err != nil {
-				return nil, err
-			}
-			constant = p.concatLiteral(constant, lit)
-		}
-	}
-
 	return &EnumValueOption{
 		OptionName: optionName,
 		Constant:   constant,
 	}, nil
 }
-
-func (p *Parser) concatLiteral(base, next string) string {
-	unQuoteNext, _, err := p.unQuote(next)
-	if err != nil {
-		return base + next
-	}
-
-	unQuoteBase, quote, err := p.unQuote(base)
-	if err != nil {
-		return base + next
-	}
-
-	return string(quote) + unQuoteBase + unQuoteNext + string(quote)
-}
-
-func (p *Parser) unQuote(lit string) (string, rune, error) {
-	for _, c := range []rune{'\'', '"'} {
-		if strings.HasPrefix(lit, string(c)) && strings.HasSuffix(lit, string(c)) {
-			return lit[1 : len(lit)-1], c, nil
-		}
-	}
-	return lit, 0, p.unexpected("quote")
-}

parser/field.go

@@ -186,7 +186,7 @@ func (p *Parser) parseFieldOption() (*FieldOption, error) {
 			return nil, err
 		}
 	default:
-		constant, _, err = p.lex.ReadConstant()
+		constant, _, err = p.lex.ReadConstant(p.permissive)
 		if err != nil {
 			return nil, err
 		}
@@ -221,7 +221,7 @@ func (p *Parser) parseGoProtoValidatorFieldOptionConstant() (string, error) {
 		}
 		ret += p.lex.Text
 
-		constant, _, err := p.lex.ReadConstant()
+		constant, _, err := p.lex.ReadConstant(p.permissive)
 		if err != nil {
 			return "", err
 		}

parser/option.go

@@ -71,7 +71,7 @@ func (p *Parser) ParseOption() (*Option, error) {
 			return nil, err
 		}
 	default:
-		constant, _, err = p.lex.ReadConstant()
+		constant, _, err = p.lex.ReadConstant(p.permissive)
 		if err != nil {
 			return nil, err
 		}
@@ -114,7 +114,7 @@ func (p *Parser) parseCloudEndpointsOptionConstant() (string, error) {
 		}
 		ret += p.lex.Text
 
-		constant, _, err := p.lex.ReadConstant()
+		constant, _, err := p.lex.ReadConstant(p.permissive)
 		if err != nil {
 			return "", err
 		}

parser/option_test.go

@@ -110,6 +110,28 @@ option (google.api.http) = {
     post: "/v1/resources",
     body: "resource",
     rest_method_name: "insert"
+};`,
+			permissive: true,
+			wantOption: &parser.Option{
+				OptionName: "(google.api.http)",
+				Constant:   `{post:"/v1/resources",body:"resource",rest_method_name:"insert"}`,
+				Meta: meta.Meta{
+					Pos: meta.Position{
+						Offset: 1,
+						Line:   2,
+						Column: 1,
+					},
+				},
+			},
+		},
+		{
+			name: "parses multiline string literal in multi-option annotation",
+			input: `
+option (google.api.http) = {
+    post: "/v1/resources",
+    body: "res"
+		      "ource",
+    rest_method_name: "insert"
 };`,
 			permissive: true,
 			wantOption: &parser.Option{