Commit cd7826e5 authored by Rob Pike's avatar Rob Pike

First cut at the parser for the new template package.

This is not a full grammar, but the pieces are there to implement whatever we converge on.

R=rsc
CC=golang-dev
https://golang.org/cl/4629053
parent 123549ac
......@@ -7,5 +7,6 @@ include ../../../Make.inc
TARG=template
GOFILES=\
lex.go\
parse.go\
include ../../../Make.pkg
......@@ -17,22 +17,77 @@ type item struct {
val string
}
func (i item) String() string {
switch i.typ {
case itemEOF:
return "EOF"
case itemError:
return i.val
}
if len(i.val) > 10 {
return fmt.Sprintf("%.10q...", i.val)
}
return fmt.Sprintf("%q", i.val)
}
// itemType identifies the type of lex item.
type itemType int
const (
itemError itemType = iota // error occurred; value is text of error
itemText // plain text
itemLeftMeta // left meta-string
itemRightMeta // right meta-string
itemPipe // pipe symbol
itemIdentifier // alphanumeric identifier
itemNumber // number
itemRawString // raw quoted string (includes quotes)
itemString // quoted string (includes quotes)
itemError itemType = iota // error occurred; value is text of error
itemDot // the cursor, spelled '.'.
itemEOF
itemElse // else keyword
itemEnd // end keyword
itemField // alphanumeric identifier, starting with '.'.
itemIdentifier // alphanumeric identifier
itemIf // if keyword
itemLeftMeta // left meta-string
itemNumber // number
itemPipe // pipe symbol
itemRange // range keyword
itemRawString // raw quoted string (includes quotes)
itemRightMeta // right meta-string
itemString // quoted string (includes quotes)
itemText // plain text
)
// Make the types prettyprint.
var itemName = map[itemType]string{
itemError: "error",
itemDot: ".",
itemEOF: "EOF",
itemElse: "else",
itemEnd: "end",
itemField: "field",
itemIdentifier: "identifier",
itemIf: "if",
itemLeftMeta: "left meta",
itemNumber: "number",
itemPipe: "pipe",
itemRange: "range",
itemRawString: "raw string",
itemRightMeta: "rightMeta",
itemString: "string",
itemText: "text",
}
func (i itemType) String() string {
s := itemName[i]
if s == "" {
return fmt.Sprintf("item%d", int(i))
}
return s
}
var key = map[string]itemType{
".": itemDot,
"else": itemElse,
"end": itemEnd,
"if": itemIf,
"range": itemRange,
}
const eof = -1
// stateFn represents the state of the scanner as a function that returns the next state.
......@@ -51,6 +106,7 @@ type lexer struct {
// next returns the next rune in the input.
func (l *lexer) next() (rune int) {
if l.pos >= len(l.input) {
l.width = 0
return eof
}
rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
......@@ -72,12 +128,11 @@ func (l *lexer) backup() {
// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
start := l.start
l.items <- item{t, l.input[l.start:l.pos]}
l.start = l.pos
l.items <- item{t, l.input[start:l.pos]}
}
// ignore discards whatever input is before this point.
// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
l.start = l.pos
}
......@@ -106,13 +161,12 @@ func (l *lexer) lineNumber() int {
// error returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.run.
func (l *lexer) error(format string, args ...interface{}) stateFn {
format = fmt.Sprintf("%s:%d %s", l.name, l.lineNumber(), format)
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
l.items <- item{itemError, fmt.Sprintf(format, args...)}
return nil
}
// run lexes the input by execute state functions until nil.
// run lexes the input by executing state functions until nil.
func (l *lexer) run() {
for state := lexText; state != nil; {
state = state(l)
......@@ -121,14 +175,14 @@ func (l *lexer) run() {
}
// lex launches a new scanner and returns the channel of items.
func lex(name, input string) chan item {
func lex(name, input string) (*lexer, chan item) {
l := &lexer{
name: name,
input: input,
items: make(chan item),
}
go l.run()
return l.items
return l, l.items
}
// state functions
......@@ -182,7 +236,7 @@ func lexInsideAction(l *lexer) stateFn {
}
switch r := l.next(); {
case r == eof || r == '\n':
return l.error("unclosed action")
return l.errorf("unclosed action")
case isSpace(r):
l.ignore()
case r == '|':
......@@ -191,20 +245,29 @@ func lexInsideAction(l *lexer) stateFn {
return lexQuote
case r == '`':
return lexRawQuote
case r == '+' || r == '-' || r == '.' || ('0' <= r && r <= '9'):
case r == '.':
// special look-ahead for ".field" so we don't break l.backup().
if l.pos < len(l.input) {
r := l.input[l.pos]
if r < '0' || '9' < r {
return lexIdentifier // itemDot comes from the keyword table.
}
}
fallthrough // '.' can start a number.
case r == '+' || r == '-' || ('0' <= r && r <= '9'):
l.backup()
return lexNumber
case isAlphaNumeric(r):
l.backup()
return lexIdentifier
default:
return l.error("unrecognized character in action: %#U", r)
return l.errorf("unrecognized character in action: %#U", r)
}
}
return nil
}
// lexIdentifier scans an alphanumeric.
// lexIdentifier scans an alphanumeric or field.
func lexIdentifier(l *lexer) stateFn {
Loop:
for {
......@@ -213,7 +276,15 @@ Loop:
// absorb
default:
l.backup()
l.emit(itemIdentifier)
word := l.input[l.start:l.pos]
switch {
case key[word] != itemError:
l.emit(key[word])
case word[0] == '.':
l.emit(itemField)
default:
l.emit(itemIdentifier)
}
break Loop
}
}
......@@ -246,7 +317,7 @@ func lexNumber(l *lexer) stateFn {
// Next thing mustn't be alphanumeric.
if isAlphaNumeric(l.peek()) {
l.next()
return l.error("bad number syntax: %q", l.input[l.start:l.pos])
return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
}
l.emit(itemNumber)
return lexInsideAction
......@@ -263,7 +334,7 @@ Loop:
}
fallthrough
case eof, '\n':
return l.error("unterminated quoted string")
return l.errorf("unterminated quoted string")
case '"':
break Loop
}
......@@ -278,7 +349,7 @@ Loop:
for {
switch l.next() {
case eof, '\n':
return l.error("unterminated raw quoted string")
return l.errorf("unterminated raw quoted string")
case '`':
break Loop
}
......
......@@ -5,33 +5,10 @@
package template
import (
"fmt"
"reflect"
"testing"
)
// Make the types prettyprint.
var itemName = map[itemType]string{
itemError: "Error",
itemText: "Text",
itemLeftMeta: "LeftMeta",
itemRightMeta: "RightMeta",
itemPipe: "Pipe",
itemIdentifier: "Identifier",
itemNumber: "Number",
itemRawString: "RawString",
itemString: "String",
itemEOF: "EOF",
}
func (i itemType) String() string {
s := itemName[i]
if s == "" {
return fmt.Sprintf("item%d", int(i))
}
return s
}
type lexTest struct {
name string
input string
......@@ -42,6 +19,7 @@ var (
tEOF = item{itemEOF, ""}
tLeft = item{itemLeftMeta, "{{"}
tRight = item{itemRightMeta, "}}"}
tRange = item{itemRange, "range"}
tPipe = item{itemPipe, "|"}
tFor = item{itemIdentifier, "for"}
tQuote = item{itemString, `"abc \n\t\" "`}
......@@ -68,6 +46,25 @@ var lexTests = []lexTest{
tRight,
tEOF,
}},
{"dots", "{{.x . .2 .x.y }}", []item{
tLeft,
{itemField, ".x"},
{itemDot, "."},
{itemNumber, ".2"},
{itemField, ".x"},
{itemField, ".y"},
tRight,
tEOF,
}},
{"keywords", "{{range if else end}}", []item{
tLeft,
{itemRange, "range"},
{itemIf, "if"},
{itemElse, "else"},
{itemEnd, "end"},
tRight,
tEOF,
}},
{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
{itemText, "intro "},
tLeft,
......@@ -88,29 +85,35 @@ var lexTests = []lexTest{
{"badchar", "#{{#}}", []item{
{itemText, "#"},
tLeft,
{itemError, "badchar:1 unrecognized character in action: U+0023 '#'"},
{itemError, "unrecognized character in action: U+0023 '#'"},
}},
{"unclosed action", "{{\n}}", []item{
tLeft,
{itemError, "unclosed action:2 unclosed action"},
{itemError, "unclosed action"},
}},
{"EOF in action", "{{range", []item{
tLeft,
tRange,
{itemError, "unclosed action"},
}},
{"unclosed quote", "{{\"\n\"}}", []item{
tLeft,
{itemError, "unclosed quote:2 unterminated quoted string"},
{itemError, "unterminated quoted string"},
}},
{"unclosed raw quote", "{{`xx\n`}}", []item{
tLeft,
{itemError, "unclosed raw quote:2 unterminated raw quoted string"},
{itemError, "unterminated raw quoted string"},
}},
{"bad number", "{{3k}}", []item{
tLeft,
{itemError, `bad number:1 bad number syntax: "3k"`},
{itemError, `bad number syntax: "3k"`},
}},
}
// collect gathers the emitted items into a slice.
func collect(t *lexTest) (items []item) {
for i := range lex(t.name, t.input) {
_, tokens := lex(t.name, t.input)
for i := range tokens {
items = append(items, i)
}
return
......@@ -120,7 +123,7 @@ func TestLex(t *testing.T) {
for _, test := range lexTests {
items := collect(&test)
if !reflect.DeepEqual(items, test.items) {
t.Errorf("%s: got\n\t%v; expected\n\t%v", test.name, items, test.items)
t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
}
}
}
This diff is collapsed.
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package template
import (
"fmt"
"testing"
)
const dumpErrors = true
type numberTest struct {
text string
isInt bool
isUint bool
isFloat bool
imaginary bool
int64
uint64
float64
}
var numberTests = []numberTest{
// basics
{"0", true, true, true, false, 0, 0, 0},
{"73", true, true, true, false, 73, 73, 73},
{"-73", true, false, true, false, -73, 0, -73},
{"+73", true, false, true, false, 73, 0, 73},
{"100", true, true, true, false, 100, 100, 100},
{"1e9", true, true, true, false, 1e9, 1e9, 1e9},
{"-1e9", true, false, true, false, -1e9, 0, -1e9},
{"-1.2", false, false, true, false, 0, 0, -1.2},
{"1e19", false, true, true, false, 0, 1e19, 1e19},
{"-1e19", false, false, true, false, 0, 0, -1e19},
{"4i", false, false, true, true, 0, 0, 4},
// funny bases
{"0123", true, true, true, false, 0123, 0123, 0123},
{"-0x0", true, false, true, false, 0, 0, 0},
{"0xdeadbeef", true, true, true, false, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef},
// some broken syntax
{text: "+-2"},
{text: "0x123."},
{text: "1e."},
{text: "0xi."},
}
func TestNumberParse(t *testing.T) {
for _, test := range numberTests {
n, err := newNumber(test.text)
ok := test.isInt || test.isUint || test.isFloat
if ok && err != nil {
t.Errorf("unexpected error for %q", test.text)
continue
}
if !ok && err == nil {
t.Errorf("expected error for %q", test.text)
continue
}
if !ok {
continue
}
if n.imaginary != test.imaginary {
t.Errorf("imaginary incorrect for %q; should be %t", test.text, test.imaginary)
}
if test.isInt {
if !n.isInt {
t.Errorf("expected integer for %q", test.text)
}
if n.int64 != test.int64 {
t.Errorf("int64 for %q should be %d is %d", test.text, test.int64, n.int64)
}
} else if n.isInt {
t.Errorf("did not expect integer for %q", test.text)
}
if test.isUint {
if !n.isUint {
t.Errorf("expected unsigned integer for %q", test.text)
}
if n.uint64 != test.uint64 {
t.Errorf("uint64 for %q should be %d is %d", test.text, test.uint64, n.uint64)
}
} else if n.isUint {
t.Errorf("did not expect unsigned integer for %q", test.text)
}
if test.isFloat {
if !n.isFloat {
t.Errorf("expected float for %q", test.text)
}
if n.float64 != test.float64 {
t.Errorf("float64 for %q should be %g is %g", test.text, test.float64, n.float64)
}
} else if n.isFloat {
t.Errorf("did not expect float for %q", test.text)
}
}
}
func num(s string) *numberNode {
n, err := newNumber(s)
if err != nil {
panic(err)
}
return n
}
type parseTest struct {
name string
input string
ok bool
result string
}
const (
noError = true
hasError = false
)
var parseTests = []parseTest{
{"empty", "", noError,
`[]`},
{"spaces", " \t\n", noError,
`[(text: " \t\n")]`},
{"text", "some text", noError,
`[(text: "some text")]`},
{"emptyMeta", "{{}}", noError,
`[(action: [])]`},
{"simple command", "{{hello}}", noError,
`[(action: [(command: [I=hello])])]`},
{"multi-word command", "{{hello world}}", noError,
`[(action: [(command: [I=hello I=world])])]`},
{"multi-word command with number", "{{hello 80}}", noError,
`[(action: [(command: [I=hello N=80])])]`},
{"multi-word command with string", "{{hello `quoted text`}}", noError,
"[(action: [(command: [I=hello S=`quoted text`])])]"},
{"pipeline", "{{hello|world}}", noError,
`[(action: [(command: [I=hello]) (command: [I=world])])]`},
{"simple range", "{{range .x}}hello{{end}}", noError,
`[({{range F=.x}} [(text: "hello")])]`},
{"nested range", "{{range .x}}hello{{range .y}}goodbye{{end}}{{end}}", noError,
`[({{range F=.x}} [(text: "hello")({{range F=.y}} [(text: "goodbye")])])]`},
{"range with else", "{{range .x}}true{{else}}false{{end}}", noError,
`[({{range F=.x}} [(text: "true")] {{else}} [(text: "false")])]`},
// Errors.
{"unclosed action", "hello{{range", hasError, ""},
{"not a field", "hello{{range x}}{{end}}", hasError, ""},
{"missing end", "hello{{range .x}}", hasError, ""},
{"missing end after else", "hello{{range .x}}{{else}}", hasError, ""},
}
func TestParse(t *testing.T) {
for _, test := range parseTests {
tmpl := New(test.name)
err := tmpl.Parse(test.input)
switch {
case err == nil && !test.ok:
t.Errorf("%q: expected error; got none", test.name)
continue
case err != nil && test.ok:
t.Errorf("%q: unexpected error: %v", test.name, err)
continue
case err != nil && !test.ok:
// expected error, got one
if dumpErrors {
fmt.Printf("%s: %s\n\t%s\n", test.name, test.input, err)
}
continue
}
result := tmpl.root.String()
if result != test.result {
t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.result)
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment