gob: better debugging, commentary

Re-implement the debugging helper to be independent of the existing
implementation.  This is preparatory to a rewrite to clean up issue 1416.
Include a definition of the grammar of the data stream.

R=rsc
CC=golang-dev
https://golang.org/cl/3970045

src/pkg/gob/codec_test.go

@@ -1307,6 +1307,31 @@ func TestUnexportedFields(t *testing.T) {
 	}
 }
 
+var singletons = []interface{}{
+	true,
+	7,
+	3.2,
+	"hello",
+	[3]int{11, 22, 33},
+	[]float32{0.5, 0.25, 0.125},
+	map[string]int{"one": 1, "two": 2},
+}
+
+func TestDebugSingleton(t *testing.T) {
+	if debugFunc == nil {
+		return
+	}
+	b := new(bytes.Buffer)
+	// Accumulate a number of values and print them out all at once.
+	for _, x := range singletons {
+		err := NewEncoder(b).Encode(x)
+		if err != nil {
+			t.Fatal("encode:", err)
+		}
+	}
+	debugFunc(b)
+}
+
 // A type that won't be defined in the gob until we send it in an interface value.
 type OnTheFly struct {
 	A int
@@ -1325,7 +1350,7 @@ type DT struct {
 	S     []string
 }
 
-func TestDebug(t *testing.T) {
+func TestDebugStruct(t *testing.T) {
 	if debugFunc == nil {
 		return
 	}

src/pkg/gob/decode.go

@@ -49,14 +49,15 @@ func overflow(name string) os.ErrorString {
 
 // decodeUintReader reads an encoded unsigned integer from an io.Reader.
 // Used only by the Decoder to read the message length.
-func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
-	_, err = r.Read(buf[0:1])
+func decodeUintReader(r io.Reader, buf []byte) (x uint64, width int, err os.Error) {
+	width = 1
+	_, err = r.Read(buf[0:width])
 	if err != nil {
 		return
 	}
 	b := buf[0]
 	if b <= 0x7f {
-		return uint64(b), nil
+		return uint64(b), width, nil
 	}
 	nb := -int(int8(b))
 	if nb > uint64Size {
@@ -75,6 +76,7 @@ func decodeUintReader(r io.Reader, buf []byte) (x uint64, err os.Error) {
 	for i := 0; i < n; i++ {
 		x <<= 8
 		x |= uint64(buf[i])
+		width++
 	}
 	return
 }

src/pkg/gob/decoder.go

@@ -58,7 +58,7 @@ func (dec *Decoder) recvType(id typeId) {
 	dec.wireType[id] = wire
 
 	// Load the next parcel.
-	dec.recv()
+	dec.recvMessage()
 }
 
 // Decode reads the next value from the connection and stores
@@ -76,23 +76,28 @@ func (dec *Decoder) Decode(e interface{}) os.Error {
 	return dec.DecodeValue(value)
 }
 
-// recv reads the next count-delimited item from the input. It is the converse
-// of Encoder.send.
-func (dec *Decoder) recv() {
+// recvMessage reads the next count-delimited item from the input. It is the converse
+// of Encoder.writeMessage.
+func (dec *Decoder) recvMessage() {
 	// Read a count.
 	var nbytes uint64
-	nbytes, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
+	nbytes, _, dec.err = decodeUintReader(dec.r, dec.countBuf[0:])
 	if dec.err != nil {
 		return
 	}
+	dec.readMessage(int(nbytes), dec.r)
+}
+
+// readMessage reads the next nbytes bytes from the input.
+func (dec *Decoder) readMessage(nbytes int, r io.Reader) {
 	// Allocate the buffer.
-	if nbytes > uint64(len(dec.buf)) {
+	if nbytes > len(dec.buf) {
 		dec.buf = make([]byte, nbytes+1000)
 	}
 	dec.byteBuffer = bytes.NewBuffer(dec.buf[0:nbytes])
 
 	// Read the data
-	_, dec.err = io.ReadFull(dec.r, dec.buf[0:nbytes])
+	_, dec.err = io.ReadFull(r, dec.buf[0:nbytes])
 	if dec.err != nil {
 		if dec.err == os.EOF {
 			dec.err = io.ErrUnexpectedEOF
@@ -103,7 +108,7 @@ func (dec *Decoder) recv() {
 
 // decodeValueFromBuffer grabs the next value from the input. The Decoder's
 // buffer already contains data.  If the next item in the buffer is a type
-// descriptor, it may be necessary to reload the buffer, but recvType does that.
+// descriptor, it will be necessary to reload the buffer; recvType does that.
 func (dec *Decoder) decodeValueFromBuffer(value reflect.Value, ignoreInterfaceValue, countPresent bool) {
 	for dec.state.b.Len() > 0 {
 		// Receive a type id.
@@ -150,7 +155,7 @@ func (dec *Decoder) DecodeValue(value reflect.Value) os.Error {
 	defer dec.mutex.Unlock()
 
 	dec.err = nil
-	dec.recv()
+	dec.recvMessage()
 	if dec.err != nil {
 		return dec.err
 	}

src/pkg/gob/doc.go

@@ -219,6 +219,54 @@ be predefined or be defined before the value in the stream.
 */
 package gob
 
+/*
+Grammar:
+
+Tokens starting with a lower case letter are terminals; int(n)
+and uint(n) represent the signed/unsigned encodings of the value n.
+
+GobStream:
+	DelimitedMessage*
+DelimitedMessage:
+	uint(lengthOfMessage) Message
+Message:
+	TypeSequence TypedValue
+TypeSequence
+	(TypeDefinition DelimitedTypeDefinition*)?
+DelimitedTypeDefinition:
+	uint(lengthOfTypeDefinition) TypeDefinition
+TypedValue:
+	int(typeId) Value
+TypeDefinition:
+	int(-typeId) encodingOfWireType
+Value:
+	ConcreteValue | InterfaceValue
+ConcreteValue:
+	SingletonValue | StructValue
+SingletonValue:
+	int(0) FieldValue
+InterfaceValue:
+	NilInterfaceValue | NonNilInterfaceValue
+NilInterfaceValue:
+	uint(0)
+NonNilInterfaceValue:
+	ConcreteTypeName TypeSequence InterfaceContents
+ConcreteTypeName:
+	uint(lengthOfName) [already read=n] name
+InterfaceContents:
+	int(concreteTypeId) DelimitedValue
+DelimitedValue:
+	uint(length) Value
+ArrayValue:
+	uint(n) Value*n [n elements]
+MapValue:
+	uint(n) (Value Value)*n  [n (key, value) pairs]
+SliceValue:
+	uint(n) Value*n [n elements]
+StructValue:
+	(uint(fieldDelta) FieldValue)*
+*/
+
 /*
 For implementers and the curious, here is an encoded example.  Given
 	type Point struct {x, y int}