Light-weight scanner for general use.

Emphasis on minimal interface and fast scanning.
Recognizes all Go literals by default. Easily
configurable to recognize different whitespace
characters and tokens.

Provides detailed position information for each
token.

R=rsc, r
CC=golang-dev
https://golang.org/cl/181160

src/pkg/Makefile

@@ -99,6 +99,7 @@ DIRS=\
 	regexp\
 	rpc\
 	runtime\
+	scanner\
 	sort\
 	strconv\
 	strings\

src/pkg/scanner/Makefile

+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../Make.$(GOARCH)
+
+TARG=scanner
+GOFILES=\
+	scanner.go\
+
+include ../../Make.pkg