bufio: new Scanner interface

Add a new, simple interface for scanning (probably textual) data,
based on a new type called Scanner. It does its own internal buffering,
so should be plausibly efficient even without injecting a bufio.Reader.
The format of the input is defined by a "split function", by default
splitting into lines. Other implemented split functions include single
bytes, single runes, and space-separated words.

Here's the loop to scan stdin as a file of lines:

        s := bufio.NewScanner(os.Stdin)
        for s.Scan() {
                fmt.Printf("%s\n", s.Bytes())
        }
        if s.Err() != nil {
                log.Fatal(s.Err())
        }

While we're dealing with spaces, define what space means to strings.Fields.

Fixes #4802.

R=adg, rogpeppe, bradfitz, rsc
CC=golang-dev
https://golang.org/cl/7322088

src/pkg/bufio/bufio_test.go

@@ -953,7 +953,7 @@ func TestNegativeRead(t *testing.T) {
 			t.Fatal("read did not panic")
 		case error:
 			if !strings.Contains(err.Error(), "reader returned negative count from Read") {
-				t.Fatal("wrong panic: %v", err)
+				t.Fatalf("wrong panic: %v", err)
 			}
 		default:
 			t.Fatalf("unexpected panic value: %T(%v)", err, err)

src/pkg/bufio/export_test.go

+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bufio
+
+// Exported for testing only.
+import (
+	"unicode/utf8"
+)
+
+var IsSpace = isSpace
+
+func (s *Scanner) MaxTokenSize(n int) {
+	if n < utf8.UTFMax || n > 1e9 {
+		panic("bad max token size")
+	}
+	if n < len(s.buf) {
+		s.buf = make([]byte, n)
+	}
+	s.maxTokenSize = n
+}
+
+// ErrOrEOF is like Err, but returns EOF. Used to test a corner case.
+func (s *Scanner) ErrOrEOF() error {
+	return s.err
+}

src/pkg/bufio/scan_test.go

+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bufio_test
+
+import (
+	. "bufio"
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Test white space table matches the Unicode definition.
+func TestSpace(t *testing.T) {
+	for r := rune(0); r <= utf8.MaxRune; r++ {
+		if IsSpace(r) != unicode.IsSpace(r) {
+			t.Fatalf("white space property disagrees: %#U should be %t", r, unicode.IsSpace(r))
+		}
+	}
+}
+
+var scanTests = []string{
+	"",
+	"a",
+	"¼",
+	"☹",
+	"\x81",   // UTF-8 error
+	"\uFFFD", // correctly encoded RuneError
+	"abcdefgh",
+	"abc def\n\t\tgh    ",
+	"abc¼☹\x81\uFFFD日本語\x82abc",
+}
+
+func TestScanByte(t *testing.T) {
+	for n, test := range scanTests {
+		buf := bytes.NewBufferString(test)
+		s := NewScanner(buf)
+		s.Split(ScanBytes)
+		var i int
+		for i = 0; s.Scan(); i++ {
+			if b := s.Bytes(); len(b) != 1 || b[0] != test[i] {
+				t.Errorf("#%d: %d: expected %q got %q", n, i, test, b)
+			}
+		}
+		if i != len(test) {
+			t.Errorf("#%d: termination expected at %d; got %d", n, len(test), i)
+		}
+		err := s.Err()
+		if err != nil {
+			t.Errorf("#%d: %v", n, err)
+		}
+	}
+}
+
+// Test that the rune splitter returns same sequence of runes (not bytes) as for range string.
+func TestScanRune(t *testing.T) {
+	for n, test := range scanTests {
+		buf := bytes.NewBufferString(test)
+		s := NewScanner(buf)
+		s.Split(ScanRunes)
+		var i, runeCount int
+		var expect rune
+		// Use a string range loop to validate the sequence of runes.
+		for i, expect = range string(test) {
+			if !s.Scan() {
+				break
+			}
+			runeCount++
+			got, _ := utf8.DecodeRune(s.Bytes())
+			if got != expect {
+				t.Errorf("#%d: %d: expected %q got %q", n, i, expect, got)
+			}
+		}
+		if s.Scan() {
+			t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
+		}
+		testRuneCount := utf8.RuneCountInString(test)
+		if runeCount != testRuneCount {
+			t.Errorf("#%d: termination expected at %d; got %d", n, testRuneCount, runeCount)
+		}
+		err := s.Err()
+		if err != nil {
+			t.Errorf("#%d: %v", n, err)
+		}
+	}
+}
+
+var wordScanTests = []string{
+	"",
+	" ",
+	"\n",
+	"a",
+	" a ",
+	"abc def",
+	" abc def ",
+	" abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n",
+}
+
+// Test that the word splitter returns the same data as strings.Fields.
+func TestScanWords(t *testing.T) {
+	for n, test := range wordScanTests {
+		buf := bytes.NewBufferString(test)
+		s := NewScanner(buf)
+		s.Split(ScanWords)
+		words := strings.Fields(test)
+		var wordCount int
+		for wordCount = 0; wordCount < len(words); wordCount++ {
+			if !s.Scan() {
+				break
+			}
+			got := s.Text()
+			if got != words[wordCount] {
+				t.Errorf("#%d: %d: expected %q got %q", n, wordCount, words[wordCount], got)
+			}
+		}
+		if s.Scan() {
+			t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
+		}
+		if wordCount != len(words) {
+			t.Errorf("#%d: termination expected at %d; got %d", n, len(words), wordCount)
+		}
+		err := s.Err()
+		if err != nil {
+			t.Errorf("#%d: %v", n, err)
+		}
+	}
+}
+
+// slowReader is a reader that returns only a few bytes at a time, to test the incremental
+// reads in Scanner.Scan.
+type slowReader struct {
+	max int
+	buf *bytes.Buffer
+}
+
+func (sr *slowReader) Read(p []byte) (n int, err error) {
+	if len(p) > sr.max {
+		p = p[0:sr.max]
+	}
+	return sr.buf.Read(p)
+}
+
+// genLine writes to buf a predictable but non-trivial line of text of length
+// n, including the terminal newline and an occasional carriage return.
+// If addNewline is false, the \r and \n are not emitted.
+func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) {
+	buf.Reset()
+	doCR := lineNum%5 == 0
+	if doCR {
+		n--
+	}
+	for i := 0; i < n-1; i++ { // Stop early for \n.
+		c := 'a' + byte(lineNum+i)
+		if c == '\n' || c == '\r' { // Don't confuse us.
+			c = 'N'
+		}
+		buf.WriteByte(c)
+	}
+	if addNewline {
+		if doCR {
+			buf.WriteByte('\r')
+		}
+		buf.WriteByte('\n')
+	}
+	return
+}
+
+// Test the line splitter, including some carriage returns but no long lines.
+func TestScanLongLines(t *testing.T) {
+	const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
+	// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
+	tmp := new(bytes.Buffer)
+	buf := new(bytes.Buffer)
+	lineNum := 0
+	j := 0
+	for i := 0; i < 2*smallMaxTokenSize; i++ {
+		genLine(tmp, lineNum, j, true)
+		if j < smallMaxTokenSize {
+			j++
+		} else {
+			j--
+		}
+		buf.Write(tmp.Bytes())
+		lineNum++
+	}
+	s := NewScanner(&slowReader{1, buf})
+	s.Split(ScanLines)
+	s.MaxTokenSize(smallMaxTokenSize)
+	j = 0
+	for lineNum := 0; s.Scan(); lineNum++ {
+		genLine(tmp, lineNum, j, false)
+		if j < smallMaxTokenSize {
+			j++
+		} else {
+			j--
+		}
+		line := tmp.String() // We use the string-valued token here, for variety.
+		if s.Text() != line {
+			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Text(), line)
+		}
+	}
+	err := s.Err()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Test that the line splitter errors out on a long line.
+func TestScanLineTooLong(t *testing.T) {
+	const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
+	// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
+	tmp := new(bytes.Buffer)
+	buf := new(bytes.Buffer)
+	lineNum := 0
+	j := 0
+	for i := 0; i < 2*smallMaxTokenSize; i++ {
+		genLine(tmp, lineNum, j, true)
+		j++
+		buf.Write(tmp.Bytes())
+		lineNum++
+	}
+	s := NewScanner(&slowReader{3, buf})
+	s.Split(ScanLines)
+	s.MaxTokenSize(smallMaxTokenSize)
+	j = 0
+	for lineNum := 0; s.Scan(); lineNum++ {
+		genLine(tmp, lineNum, j, false)
+		if j < smallMaxTokenSize {
+			j++
+		} else {
+			j--
+		}
+		line := tmp.Bytes()
+		if !bytes.Equal(s.Bytes(), line) {
+			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
+		}
+	}
+	err := s.Err()
+	if err != ErrTooLong {
+		t.Fatalf("expected ErrTooLong; got %s", err)
+	}
+}
+
+// Test that the line splitter handles a final line without a newline.
+func testNoNewline(text string, lines []string, t *testing.T) {
+	buf := bytes.NewBufferString(text)
+	s := NewScanner(&slowReader{7, buf})
+	s.Split(ScanLines)
+	for lineNum := 0; s.Scan(); lineNum++ {
+		line := lines[lineNum]
+		if s.Text() != line {
+			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
+		}
+	}
+	err := s.Err()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+var noNewlineLines = []string{
+	"abcdefghijklmn\nopqrstuvwxyz",
+}
+
+// Test that the line splitter handles a final line without a newline.
+func TestScanLineNoNewline(t *testing.T) {
+	const text = "abcdefghijklmn\nopqrstuvwxyz"
+	lines := []string{
+		"abcdefghijklmn",
+		"opqrstuvwxyz",
+	}
+	testNoNewline(text, lines, t)
+}
+
+// Test that the line splitter handles a final line with a carriage return but nonewline.
+func TestScanLineReturnButNoNewline(t *testing.T) {
+	const text = "abcdefghijklmn\nopqrstuvwxyz\r"
+	lines := []string{
+		"abcdefghijklmn",
+		"opqrstuvwxyz",
+	}
+	testNoNewline(text, lines, t)
+}
+
+// Test that the line splitter handles a final empty line.
+func TestScanLineEmptyFinalLine(t *testing.T) {
+	const text = "abcdefghijklmn\nopqrstuvwxyz\n\n"
+	lines := []string{
+		"abcdefghijklmn",
+		"opqrstuvwxyz",
+		"",
+	}
+	testNoNewline(text, lines, t)
+}
+
+// Test that the line splitter handles a final empty line with a carriage return but no newline.
+func TestScanLineEmptyFinalLineWithCR(t *testing.T) {
+	const text = "abcdefghijklmn\nopqrstuvwxyz\n\r"
+	lines := []string{
+		"abcdefghijklmn",
+		"opqrstuvwxyz",
+		"",
+	}
+	testNoNewline(text, lines, t)
+}
+
+var testError = errors.New("testError")
+
+// Test the correct error is returned when the split function errors out.
+func TestSplitError(t *testing.T) {
+	// Create a split function that delivers a little data, then a predictable error.
+	numSplits := 0
+	const okCount = 7
+	errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+		if atEOF {
+			panic("didn't get enough data")
+		}
+		if numSplits >= okCount {
+			return 0, nil, testError
+		}
+		numSplits++
+		return 1, data[0:1], nil
+	}
+	// Read the data.
+	const text = "abcdefghijklmnopqrstuvwxyz"
+	buf := bytes.NewBufferString(text)
+	s := NewScanner(&slowReader{1, buf})
+	s.Split(errorSplit)
+	var i int
+	for i = 0; s.Scan(); i++ {
+		if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] {
+			t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0])
+		}
+	}
+	// Check correct termination location and error.
+	if i != okCount {
+		t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i)
+	}
+	err := s.Err()
+	if err != testError {
+		t.Fatalf("expected %q got %v", testError, err)
+	}
+}
+
+// Test that an EOF is overridden by a user-generated scan error.
+func TestErrAtEOF(t *testing.T) {
+	s := NewScanner(strings.NewReader("1 2 33"))
+	// This spitter will fail on last entry, after s.err==EOF.
+	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+		advance, token, err = ScanWords(data, atEOF)
+		if len(token) > 1 {
+			if s.ErrOrEOF() != io.EOF {
+				t.Fatal("not testing EOF")
+			}
+			err = testError
+		}
+		return
+	}
+	s.Split(split)
+	for s.Scan() {
+	}
+	if s.Err() != testError {
+		t.Fatal("wrong error:", s.Err())
+	}
+}

src/pkg/strings/strings.go

@@ -305,7 +305,8 @@ func SplitAfter(s, sep string) []string {
 }
 
 // Fields splits the string s around each instance of one or more consecutive white space
-// characters, returning an array of substrings of s or an empty list if s contains only white space.
+// characters, as defined by unicode.IsSpace, returning an array of substrings of s or an
+// empty list if s contains only white space.
 func Fields(s string) []string {
 	return FieldsFunc(s, unicode.IsSpace)
 }