Commit 5dc6c808 authored by Robert Griesemer's avatar Robert Griesemer

report an error for illegal octal numbers instead of treating them as floats

added more test cases
some capitalization cleanups

R=rsc
CC=golang-dev
https://golang.org/cl/180085
parent c5f41cc5
...@@ -271,10 +271,11 @@ func (S *Scanner) scanMantissa(base int) { ...@@ -271,10 +271,11 @@ func (S *Scanner) scanMantissa(base int) {
} }
func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token { func (S *Scanner) scanNumber(pos token.Position, seenDecimalPoint bool) token.Token {
// digitVal(S.ch) < 10
tok := token.INT tok := token.INT
if seen_decimal_point { if seenDecimalPoint {
tok = token.FLOAT tok = token.FLOAT
S.scanMantissa(10) S.scanMantissa(10)
goto exponent goto exponent
...@@ -289,23 +290,29 @@ func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token { ...@@ -289,23 +290,29 @@ func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
S.scanMantissa(16) S.scanMantissa(16)
} else { } else {
// octal int or float // octal int or float
seenDecimalDigit := false
S.scanMantissa(8) S.scanMantissa(8)
if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' { if S.ch == '8' || S.ch == '9' {
// float // illegal octal int or float
tok = token.FLOAT seenDecimalDigit = true
goto mantissa S.scanMantissa(10)
}
if S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
goto fraction
} }
// octal int // octal int
if seenDecimalDigit {
S.error(pos, "illegal octal number")
}
} }
goto exit goto exit
} }
mantissa:
// decimal int or float // decimal int or float
S.scanMantissa(10) S.scanMantissa(10)
fraction:
if S.ch == '.' { if S.ch == '.' {
// float
tok = token.FLOAT tok = token.FLOAT
S.next() S.next()
S.scanMantissa(10) S.scanMantissa(10)
...@@ -313,7 +320,6 @@ mantissa: ...@@ -313,7 +320,6 @@ mantissa:
exponent: exponent:
if S.ch == 'e' || S.ch == 'E' { if S.ch == 'e' || S.ch == 'E' {
// float
tok = token.FLOAT tok = token.FLOAT
S.next() S.next()
if S.ch == '-' || S.ch == '+' { if S.ch == '-' || S.ch == '+' {
...@@ -503,7 +509,7 @@ scanAgain: ...@@ -503,7 +509,7 @@ scanAgain:
} }
case digitVal(ch) < 10: case digitVal(ch) < 10:
insertSemi = true insertSemi = true
tok = S.scanNumber(false) tok = S.scanNumber(pos, false)
default: default:
S.next() // always make progress S.next() // always make progress
switch ch { switch ch {
...@@ -532,7 +538,7 @@ scanAgain: ...@@ -532,7 +538,7 @@ scanAgain:
case '.': case '.':
if digitVal(S.ch) < 10 { if digitVal(S.ch) < 10 {
insertSemi = true insertSemi = true
tok = S.scanNumber(true) tok = S.scanNumber(pos, true)
} else if S.ch == '.' { } else if S.ch == '.' {
S.next() S.next()
if S.ch == '.' { if S.ch == '.' {
......
...@@ -164,16 +164,16 @@ var tokens = [...]elt{ ...@@ -164,16 +164,16 @@ var tokens = [...]elt{
const whitespace = " \t \n\n\n" // to separate tokens const whitespace = " \t \n\n\n" // to separate tokens
type TestErrorHandler struct { type testErrorHandler struct {
t *testing.T t *testing.T
} }
func (h *TestErrorHandler) Error(pos token.Position, msg string) { func (h *testErrorHandler) Error(pos token.Position, msg string) {
h.t.Errorf("Error() called (msg = %s)", msg) h.t.Errorf("Error() called (msg = %s)", msg)
} }
func NewlineCount(s string) int { func newlineCount(s string) int {
n := 0 n := 0
for i := 0; i < len(s); i++ { for i := 0; i < len(s); i++ {
if s[i] == '\n' { if s[i] == '\n' {
...@@ -207,12 +207,12 @@ func TestScan(t *testing.T) { ...@@ -207,12 +207,12 @@ func TestScan(t *testing.T) {
for _, e := range tokens { for _, e := range tokens {
src += e.lit + whitespace src += e.lit + whitespace
} }
whitespace_linecount := NewlineCount(whitespace) whitespace_linecount := newlineCount(whitespace)
// verify scan // verify scan
index := 0 index := 0
epos := token.Position{"", 0, 1, 1} epos := token.Position{"", 0, 1, 1}
nerrors := Tokenize("", strings.Bytes(src), &TestErrorHandler{t}, ScanComments, nerrors := Tokenize("", strings.Bytes(src), &testErrorHandler{t}, ScanComments,
func(pos token.Position, tok token.Token, litb []byte) bool { func(pos token.Position, tok token.Token, litb []byte) bool {
e := elt{token.EOF, "", special} e := elt{token.EOF, "", special}
if index < len(tokens) { if index < len(tokens) {
...@@ -234,7 +234,7 @@ func TestScan(t *testing.T) { ...@@ -234,7 +234,7 @@ func TestScan(t *testing.T) {
t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class) t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
} }
epos.Offset += len(lit) + len(whitespace) epos.Offset += len(lit) + len(whitespace)
epos.Line += NewlineCount(lit) + whitespace_linecount epos.Line += newlineCount(lit) + whitespace_linecount
if tok == token.COMMENT && litb[1] == '/' { if tok == token.COMMENT && litb[1] == '/' {
// correct for unaccounted '/n' in //-style comment // correct for unaccounted '/n' in //-style comment
epos.Offset++ epos.Offset++
...@@ -249,11 +249,6 @@ func TestScan(t *testing.T) { ...@@ -249,11 +249,6 @@ func TestScan(t *testing.T) {
} }
func getTok(_ token.Position, tok token.Token, _ []byte) token.Token {
return tok
}
func checkSemi(t *testing.T, line string, mode uint) { func checkSemi(t *testing.T, line string, mode uint) {
var S Scanner var S Scanner
S.Init("TestSemis", strings.Bytes(line), nil, mode) S.Init("TestSemis", strings.Bytes(line), nil, mode)
...@@ -485,7 +480,7 @@ func TestIllegalChars(t *testing.T) { ...@@ -485,7 +480,7 @@ func TestIllegalChars(t *testing.T) {
var s Scanner var s Scanner
const src = "*?*$*@*" const src = "*?*$*@*"
s.Init("", strings.Bytes(src), &TestErrorHandler{t}, AllowIllegalChars) s.Init("", strings.Bytes(src), &testErrorHandler{t}, AllowIllegalChars)
for offs, ch := range src { for offs, ch := range src {
pos, tok, lit := s.Scan() pos, tok, lit := s.Scan()
if pos.Offset != offs { if pos.Offset != offs {
...@@ -540,3 +535,74 @@ func TestStdErrorHander(t *testing.T) { ...@@ -540,3 +535,74 @@ func TestStdErrorHander(t *testing.T) {
t.Errorf("found %d errors, expected %d", v.ErrorCount(), nerrors) t.Errorf("found %d errors, expected %d", v.ErrorCount(), nerrors)
} }
} }
type errorCollector struct {
cnt int // number of errors encountered
msg string // last error message encountered
pos token.Position // last error position encountered
}
func (h *errorCollector) Error(pos token.Position, msg string) {
h.cnt++
h.msg = msg
h.pos = pos
}
func checkError(t *testing.T, src string, tok token.Token, err string) {
var s Scanner
var h errorCollector
s.Init("", strings.Bytes(src), &h, ScanComments)
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {
t.Errorf("%q: got %s, expected %s", src, tok0, tok)
}
if tok1 != token.EOF {
t.Errorf("%q: got %s, expected EOF", src, tok1)
}
cnt := 0
if err != "" {
cnt = 1
}
if h.cnt != cnt {
t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
}
if h.msg != err {
t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
}
if h.pos.Offset != 0 {
t.Errorf("%q: got offset %d, expected 0", src, h.pos.Offset)
}
}
type srcerr struct {
src string
tok token.Token
err string
}
var errors = []srcerr{
srcerr{"\"\"", token.STRING, ""},
srcerr{"\"", token.STRING, "string not terminated"},
srcerr{"/**/", token.COMMENT, ""},
srcerr{"/*", token.COMMENT, "comment not terminated"},
srcerr{"//\n", token.COMMENT, ""},
srcerr{"//", token.COMMENT, "comment not terminated"},
srcerr{"077", token.INT, ""},
srcerr{"078.", token.FLOAT, ""},
srcerr{"07801234567.", token.FLOAT, ""},
srcerr{"078e0", token.FLOAT, ""},
srcerr{"078", token.INT, "illegal octal number"},
srcerr{"07800000009", token.INT, "illegal octal number"},
}
func TestScanErrors(t *testing.T) {
for _, e := range errors {
checkError(t, e.src, e.tok, e.err)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment