encoding/line: add

I needed a way to read lines without worrying about \n and \r\n.

R=r, rsc
CC=golang-dev
https://golang.org/cl/2859041

src/pkg/Makefile

@@ -61,6 +61,7 @@ DIRS=\
 	encoding/binary\
 	encoding/git85\
 	encoding/hex\
+	encoding/line\
 	encoding/pem\
 	exec\
 	exp/datafmt\

src/pkg/encoding/line/Makefile

+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=encoding/line
+GOFILES=\
+	line.go\
+
+include ../../../Make.pkg

src/pkg/encoding/line/line.go

+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package implements a Reader which handles reading \r and \r\n
+// deliminated lines.
+package line
+
+import (
+	"io"
+	"os"
+)
+
+// Reader reads lines from an io.Reader (which may use either '\n' or
+// '\r\n').
+type Reader struct {
+	buf      []byte
+	consumed int
+	in       io.Reader
+	err      os.Error
+}
+
+func NewReader(in io.Reader, maxLineLength int) *Reader {
+	return &Reader{
+		buf:      make([]byte, 0, maxLineLength),
+		consumed: 0,
+		in:       in,
+	}
+}
+
+// ReadLine tries to return a single line, not including the end-of-line bytes.
+// If the line was found to be longer than the maximum length then isPrefix is
+// set and the beginning of the line is returned. The rest of the line will be
+// returned from future calls. isPrefix will be false when returning the last
+// fragment of the line.  The returned buffer points into the internal state of
+// the Reader and is only valid until the next call to ReadLine. ReadLine
+// either returns a non-nil line or it returns an error, never both.
+func (l *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) {
+	if l.consumed > 0 {
+		n := copy(l.buf, l.buf[l.consumed:])
+		l.buf = l.buf[:n]
+		l.consumed = 0
+	}
+
+	if len(l.buf) == 0 && l.err != nil {
+		err = l.err
+		return
+	}
+
+	scannedTo := 0
+
+	for {
+		i := scannedTo
+		for ; i < len(l.buf); i++ {
+			if l.buf[i] == '\r' && len(l.buf) > i+1 && l.buf[i+1] == '\n' {
+				line = l.buf[:i]
+				l.consumed = i + 2
+				return
+			} else if l.buf[i] == '\n' {
+				line = l.buf[:i]
+				l.consumed = i + 1
+				return
+			}
+		}
+
+		if i == cap(l.buf) {
+			line = l.buf[:i]
+			l.consumed = i
+			isPrefix = true
+			return
+		}
+
+		if l.err != nil {
+			line = l.buf
+			l.consumed = i
+			return
+		}
+
+		// We don't want to rescan the input that we just scanned.
+		// However, we need to back up one byte because the last byte
+		// could have been a '\r' and we do need to rescan that.
+		scannedTo = i
+		if scannedTo > 0 {
+			scannedTo--
+		}
+		oldLen := len(l.buf)
+		l.buf = l.buf[:cap(l.buf)]
+		n, readErr := l.in.Read(l.buf[oldLen:])
+		l.buf = l.buf[:oldLen+n]
+		if readErr != nil {
+			l.err = readErr
+		}
+	}
+	panic("unreachable")
+}

src/pkg/encoding/line/line_test.go

+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package line
+
+import (
+	"bytes"
+	"os"
+	"testing"
+)
+
+var testOutput = []byte("0123456789abcdefghijklmnopqrstuvwxy")
+var testInput = []byte("012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy")
+var testInputrn = []byte("012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\nuvw\r\nxy\r\n\n\r\n")
+
+// TestReader wraps a []byte and returns reads of a specific length.
+type testReader struct {
+	data   []byte
+	stride int
+}
+
+func (t *testReader) Read(buf []byte) (n int, err os.Error) {
+	n = t.stride
+	if n > len(t.data) {
+		n = len(t.data)
+	}
+	if n > len(buf) {
+		n = len(buf)
+	}
+	copy(buf, t.data)
+	t.data = t.data[n:]
+	if len(t.data) == 0 {
+		err = os.EOF
+	}
+	return
+}
+
+func testLineReader(t *testing.T, input []byte) {
+	for stride := 1; stride < len(input); stride++ {
+		done := 0
+		reader := testReader{input, stride}
+		l := NewReader(&reader, len(input)+1)
+		for {
+			line, isPrefix, err := l.ReadLine()
+			if len(line) > 0 && err != nil {
+				t.Errorf("ReadLine returned both data and error: %s\n")
+			}
+			if isPrefix {
+				t.Errorf("ReadLine returned prefix\n")
+			}
+			if err != nil {
+				if err != os.EOF {
+					t.Fatalf("Got unknown error: %s", err)
+				}
+				break
+			}
+			if want := testOutput[done : done+len(line)]; !bytes.Equal(want, line) {
+				t.Errorf("Bad line at stride %d: want: %x got: %x", stride, want, line)
+			}
+			done += len(line)
+		}
+		if done != len(testOutput) {
+			t.Error("ReadLine didn't return everything")
+		}
+	}
+}
+
+func TestReader(t *testing.T) {
+	testLineReader(t, testInput)
+	testLineReader(t, testInputrn)
+}
+
+func TestLineTooLong(t *testing.T) {
+	buf := bytes.NewBuffer([]byte("aaabbbcc\n"))
+	l := NewReader(buf, 3)
+	line, isPrefix, err := l.ReadLine()
+	if !isPrefix || !bytes.Equal(line, []byte("aaa")) || err != nil {
+		t.Errorf("bad result for first line: %x %s", line, err)
+	}
+	line, isPrefix, err = l.ReadLine()
+	if !isPrefix || !bytes.Equal(line, []byte("bbb")) || err != nil {
+		t.Errorf("bad result for second line: %x", line)
+	}
+	line, isPrefix, err = l.ReadLine()
+	if isPrefix || !bytes.Equal(line, []byte("cc")) || err != nil {
+		t.Errorf("bad result for third line: %x", line)
+	}
+}