Preparation for moving scanner into a lib:

- separated out token definition from scanner
- cleaned up token and scanner implementation
- added comments to exported interfaces

R=r
OCL=25665
CL=25665

usr/gri/pretty/Makefile

@@ -32,21 +32,21 @@ gds.6:	 utils.6 platform.6 compilation.6 printer.6
 
 pretty.6:	 platform.6 printer.6 compilation.6
 
-compilation.6:	 platform.6 scanner.6 parser.6 ast.6 typechecker.6
+compilation.6:	 platform.6 token.6 scanner.6 parser.6 ast.6 typechecker.6
 
-typechecker.6:	 ast.6 scanner.6
+typechecker.6:	 ast.6 token.6
 
-scanner.6:	 utils.6
+scanner.6:	 token.6 utils.6
 
-ast.6:	 scanner.6 symboltable.6
+ast.6:	 token.6 symboltable.6
 
 symboltable.6:	
 
-parser.6:	 scanner.6 ast.6 symboltable.6
+parser.6:	 token.6 scanner.6 ast.6 symboltable.6
 
 platform.6:	 utils.6
 
-printer.6:	 utils.6 scanner.6 ast.6 symboltable.6
+printer.6:	 utils.6 token.6 ast.6 symboltable.6
 
 %.6:	%.go
 	$(G) $(F) $<

usr/gri/pretty/ast.go

@@ -6,7 +6,7 @@ package AST
 
 import (
 	"vector";
-	Scanner "scanner";
+	"token";
 	SymbolTable "symboltable";
 )
 
@@ -259,7 +259,7 @@ func ExprLen(x Expr) int {
 	}
 	n := 1;
 	for {
-		if p, ok := x.(*BinaryExpr); ok && p.Tok == Scanner.COMMA {
+		if p, ok := x.(*BinaryExpr); ok && p.Tok == token.COMMA {
 			n++;
 			x = p.Y;
 		} else {
@@ -272,10 +272,10 @@ func ExprLen(x Expr) int {
 
 func ExprAt(x Expr, i int) Expr {
 	for j := 0; j < i; j++ {
-		assert(x.(*BinaryExpr).Tok == Scanner.COMMA);
+		assert(x.(*BinaryExpr).Tok == token.COMMA);
 		x = x.(*BinaryExpr).Y;
 	}
-	if t, is_binary := x.(*BinaryExpr); is_binary && t.Tok == Scanner.COMMA {
+	if t, is_binary := x.(*BinaryExpr); is_binary && t.Tok == token.COMMA {
 		x = t.X;
 	}
 	return x;
@@ -298,7 +298,7 @@ type Block struct {
 
 
 func NewBlock(pos, tok int) *Block {
-	assert(tok == Scanner.LBRACE || tok == Scanner.COLON);
+	assert(tok == token.LBRACE || tok == token.COLON);
 	b := new(Block);
 	b.Pos, b.Tok, b.List = pos, tok, vector.New(0);
 	return b;
@@ -509,7 +509,7 @@ func NewComment(pos int, text string) *Comment {
 
 
 type Program struct {
-	Pos int;  // tok is Scanner.PACKAGE
+	Pos int;  // tok is token.PACKAGE
 	Ident Expr;
 	Decls []Decl;
 	Comments *vector.Vector;

usr/gri/pretty/token.go

+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+// Defines Go tokens and basic token operations.
+
+import "strconv";
+
+const (
+	// Special tokens
+	ILLEGAL = iota;
+	EOF;
+	COMMENT;
+	
+	// Identifiers and basic type literals
+	// (these tokens stand for classes of literals)
+	literal_beg;
+	IDENT;
+	INT;
+	FLOAT;
+	CHAR;
+	STRING;
+	literal_end;
+
+	// Operators and delimiters
+	operator_beg;
+	ADD;
+	SUB;
+	MUL;
+	QUO;
+	REM;
+
+	AND;
+	OR;
+	XOR;
+	SHL;
+	SHR;
+
+	ADD_ASSIGN;
+	SUB_ASSIGN;
+	MUL_ASSIGN;
+	QUO_ASSIGN;
+	REM_ASSIGN;
+
+	AND_ASSIGN;
+	OR_ASSIGN;
+	XOR_ASSIGN;
+	SHL_ASSIGN;
+	SHR_ASSIGN;
+
+	LAND;
+	LOR;
+	ARROW;
+	INC;
+	DEC;
+
+	EQL;
+	LSS;
+	GTR;
+	ASSIGN;
+	NOT;
+
+	NEQ;
+	LEQ;
+	GEQ;
+	DEFINE;
+	ELLIPSIS;
+
+	LPAREN;
+	LBRACK;
+	LBRACE;
+	COMMA;
+	PERIOD;
+
+	RPAREN;
+	RBRACK;
+	RBRACE;
+	SEMICOLON;
+	COLON;
+	operator_end;
+
+	// Keywords
+	keyword_beg;
+	BREAK;
+	CASE;
+	CHAN;
+	CONST;
+	CONTINUE;
+
+	DEFAULT;
+	DEFER;
+	ELSE;
+	FALLTHROUGH;
+	FOR;
+
+	FUNC;
+	GO;
+	GOTO;
+	IF;
+	IMPORT;
+
+	INTERFACE;
+	MAP;
+	PACKAGE;
+	RANGE;
+	RETURN;
+
+	SELECT;
+	STRUCT;
+	SWITCH;
+	TYPE;
+	VAR;
+	keyword_end;
+)
+
+
+func TokenString(tok int) string {
+	switch tok {
+	case ILLEGAL: return "ILLEGAL";
+
+	case EOF: return "EOF";
+	case COMMENT: return "COMMENT";
+
+	case IDENT: return "IDENT";
+	case INT: return "INT";
+	case FLOAT: return "FLOAT";
+	case CHAR: return "CHAR";
+	case STRING: return "STRING";
+
+	case ADD: return "+";
+	case SUB: return "-";
+	case MUL: return "*";
+	case QUO: return "/";
+	case REM: return "%";
+
+	case AND: return "&";
+	case OR: return "|";
+	case XOR: return "^";
+	case SHL: return "<<";
+	case SHR: return ">>";
+
+	case ADD_ASSIGN: return "+=";
+	case SUB_ASSIGN: return "-=";
+	case MUL_ASSIGN: return "+=";
+	case QUO_ASSIGN: return "/=";
+	case REM_ASSIGN: return "%=";
+
+	case AND_ASSIGN: return "&=";
+	case OR_ASSIGN: return "|=";
+	case XOR_ASSIGN: return "^=";
+	case SHL_ASSIGN: return "<<=";
+	case SHR_ASSIGN: return ">>=";
+
+	case LAND: return "&&";
+	case LOR: return "||";
+	case ARROW: return "<-";
+	case INC: return "++";
+	case DEC: return "--";
+
+	case EQL: return "==";
+	case LSS: return "<";
+	case GTR: return ">";
+	case ASSIGN: return "=";
+	case NOT: return "!";
+
+	case NEQ: return "!=";
+	case LEQ: return "<=";
+	case GEQ: return ">=";
+	case DEFINE: return ":=";
+	case ELLIPSIS: return "...";
+
+	case LPAREN: return "(";
+	case LBRACK: return "[";
+	case LBRACE: return "{";
+	case COMMA: return ",";
+	case PERIOD: return ".";
+
+	case RPAREN: return ")";
+	case RBRACK: return "]";
+	case RBRACE: return "}";
+	case SEMICOLON: return ";";
+	case COLON: return ":";
+
+	case BREAK: return "break";
+	case CASE: return "case";
+	case CHAN: return "chan";
+	case CONST: return "const";
+	case CONTINUE: return "continue";
+
+	case DEFAULT: return "default";
+	case DEFER: return "defer";
+	case ELSE: return "else";
+	case FALLTHROUGH: return "fallthrough";
+	case FOR: return "for";
+
+	case FUNC: return "func";
+	case GO: return "go";
+	case GOTO: return "goto";
+	case IF: return "if";
+	case IMPORT: return "import";
+
+	case INTERFACE: return "interface";
+	case MAP: return "map";
+	case PACKAGE: return "package";
+	case RANGE: return "range";
+	case RETURN: return "return";
+
+	case SELECT: return "select";
+	case STRUCT: return "struct";
+	case SWITCH: return "switch";
+	case TYPE: return "type";
+	case VAR: return "var";
+	}
+
+	return "token(" + strconv.Itoa(tok) + ")";
+}
+
+
+const (
+	LowestPrec = -1;
+	UnaryPrec = 7;
+	HighestPrec = 8;
+)
+
+
+func Precedence(tok int) int {
+	switch tok {
+	case COLON:
+		return 0;
+	case LOR:
+		return 1;
+	case LAND:
+		return 2;
+	case ARROW:
+		return 3;
+	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
+		return 4;
+	case ADD, SUB, OR, XOR:
+		return 5;
+	case MUL, QUO, REM, SHL, SHR, AND:
+		return 6;
+	}
+	return LowestPrec;
+}
+
+
+var keywords map [string] int;
+
+func init() {
+	keywords = make(map [string] int);
+	for i := keyword_beg + 1; i < keyword_end; i++ {
+		keywords[TokenString(i)] = i;
+	}
+}
+
+
+// Map an identifier to its keyword token or IDENT (if not a keyword).
+func Lookup(ident []byte) int {
+	// TODO should not have to convert every ident into a string
+	//      for lookup - but at the moment maps of []byte don't
+	//      seem to work - gri 3/3/09
+	if tok, is_keyword := keywords[string(ident)]; is_keyword {
+		return tok;
+	}
+	return IDENT;
+}
+
+
+// Predicates
+
+// Identifiers and basic type literals
+func IsLiteral(tok int) bool {
+	return literal_beg < tok && tok < literal_end;
+}
+
+
+// Operators and delimiters
+func IsOperator(tok int) bool {
+	return operator_beg < tok && tok < operator_end;
+}
+
+func IsKeyword(tok int) bool {
+	return keyword_beg < tok && tok < keyword_end;
+}

usr/gri/pretty/typechecker.go

@@ -5,18 +5,23 @@
 package TypeChecker
 
 import (
+	"token";
 	AST "ast";
-	Scanner "scanner";
 )
 
 
+type ErrorHandler interface {
+	Error(pos int, msg string);
+}
+
+
 type state struct {
 	// setup
-	err Scanner.ErrorHandler;
+	err ErrorHandler;
 }
 
 
-func (s *state) Init(err Scanner.ErrorHandler) {
+func (s *state) Init(err ErrorHandler) {
 	s.err = err;
 }
 
@@ -54,7 +59,7 @@ func (s *state) CheckType() {
 
 /*
 func (s *state) CheckDeclaration(d *AST.Decl) {
-	if d.Tok != Scanner.FUNC && d.List != nil {
+	if d.Tok != token.FUNC && d.List != nil {
 		// group of parenthesized declarations
 		for i := 0; i < d.List.Len(); i++ {
 			s.CheckDeclaration(d.List.At(i).(*AST.Decl))
@@ -63,11 +68,11 @@ func (s *state) CheckDeclaration(d *AST.Decl) {
 	} else {
 		// single declaration
 		switch d.Tok {
-		case Scanner.IMPORT:
-		case Scanner.CONST:
-		case Scanner.VAR:
-		case Scanner.TYPE:
-		case Scanner.FUNC:
+		case token.IMPORT:
+		case token.CONST:
+		case token.VAR:
+		case token.TYPE:
+		case token.FUNC:
 		default:
 			unreachable();
 		}
@@ -85,7 +90,7 @@ func (s *state) CheckProgram(p *AST.Program) {
 
 // ----------------------------------------------------------------------------
 
-func CheckProgram(err Scanner.ErrorHandler, p *AST.Program) {
+func CheckProgram(err ErrorHandler, p *AST.Program) {
 	var s state;
 	s.Init(err);
 	s.CheckProgram(p);