encoding/json: read ahead after value consumption

Eliminates the need for an extra scanner, read undo and some other tricks.

name                    old time/op    new time/op    delta
CodeEncoder-12            1.92ms ± 0%    1.91ms ± 1%   -0.65%  (p=0.000 n=17+20)
CodeMarshal-12            2.13ms ± 2%    2.12ms ± 1%   -0.49%  (p=0.038 n=18+17)
CodeDecoder-12            8.55ms ± 2%    8.49ms ± 1%     ~     (p=0.119 n=20+18)
UnicodeDecoder-12          411ns ± 0%     422ns ± 0%   +2.77%  (p=0.000 n=19+15)
DecoderStream-12           320ns ± 1%     307ns ± 1%   -3.80%  (p=0.000 n=18+20)
CodeUnmarshal-12          9.65ms ± 3%    9.58ms ± 3%     ~     (p=0.157 n=20+20)
CodeUnmarshalReuse-12     8.54ms ± 3%    8.56ms ± 2%     ~     (p=0.602 n=20+20)
UnmarshalString-12         110ns ± 1%      87ns ± 2%  -21.53%  (p=0.000 n=16+20)
UnmarshalFloat64-12        101ns ± 1%      77ns ± 2%  -23.08%  (p=0.000 n=19+20)
UnmarshalInt64-12         94.5ns ± 2%    68.4ns ± 1%  -27.60%  (p=0.000 n=20+20)
Issue10335-12              128ns ± 1%     100ns ± 1%  -21.89%  (p=0.000 n=19+18)
Unmapped-12                427ns ± 3%     247ns ± 4%  -42.17%  (p=0.000 n=20+20)
NumberIsValid-12          23.0ns ± 0%    21.7ns ± 0%   -5.73%  (p=0.000 n=20+20)
NumberIsValidRegexp-12     641ns ± 0%     642ns ± 0%   +0.15%  (p=0.003 n=19+19)
EncoderEncode-12          56.9ns ± 0%    55.0ns ± 1%   -3.32%  (p=0.012 n=2+17)

name                    old speed      new speed      delta
CodeEncoder-12          1.01GB/s ± 1%  1.02GB/s ± 1%   +0.71%  (p=0.000 n=18+20)
CodeMarshal-12           913MB/s ± 2%   917MB/s ± 1%   +0.49%  (p=0.038 n=18+17)
CodeDecoder-12           227MB/s ± 2%   229MB/s ± 1%     ~     (p=0.110 n=20+18)
UnicodeDecoder-12       34.1MB/s ± 0%  33.1MB/s ± 0%   -2.73%  (p=0.000 n=19+19)
CodeUnmarshal-12         201MB/s ± 3%   203MB/s ± 3%     ~     (p=0.151 n=20+20)

name                    old alloc/op   new alloc/op   delta
Issue10335-12               320B ± 0%      184B ± 0%  -42.50%  (p=0.000 n=20+20)
Unmapped-12                 568B ± 0%      216B ± 0%  -61.97%  (p=0.000 n=20+20)
EncoderEncode-12           0.00B          0.00B          ~     (all equal)

name                    old allocs/op  new allocs/op  delta
Issue10335-12               4.00 ± 0%      3.00 ± 0%  -25.00%  (p=0.000 n=20+20)
Unmapped-12                 18.0 ± 0%       4.0 ± 0%  -77.78%  (p=0.000 n=20+20)
EncoderEncode-12            0.00           0.00          ~     (all equal)

Fixes #17914
Updates #20693
Updates #10335

Change-Id: I0459a52febb8b79c9a2991e69ed2614cf8740429
Reviewed-on: https://go-review.googlesource.com/47152Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/encoding/json/scanner.go

@@ -8,7 +8,7 @@ package json
 // Just about at the limit of what is reasonable to write by hand.
 // Some parts are a bit tedious, but overall it nicely factors out the
 // otherwise common code from the multiple scanning functions
-// in this package (Compact, Indent, checkValid, nextValue, etc).
+// in this package (Compact, Indent, checkValid, etc).
 //
 // This file starts with two simple examples using the scanner
 // before diving into the scanner itself.
@@ -36,35 +36,6 @@ func checkValid(data []byte, scan *scanner) error {
 	return nil
 }
 
-// nextValue splits data after the next whole JSON value,
-// returning that value and the bytes that follow it as separate slices.
-// scan is passed in for use by nextValue to avoid an allocation.
-func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) {
-	scan.reset()
-	for i, c := range data {
-		v := scan.step(scan, c)
-		if v >= scanEndObject {
-			switch v {
-			// probe the scanner with a space to determine whether we will
-			// get scanEnd on the next character. Otherwise, if the next character
-			// is not a space, scanEndTop allocates a needless error.
-			case scanEndObject, scanEndArray:
-				if scan.step(scan, ' ') == scanEnd {
-					return data[:i+1], data[i+1:], nil
-				}
-			case scanError:
-				return nil, nil, scan.err
-			case scanEnd:
-				return data[:i], data[i:], nil
-			}
-		}
-	}
-	if scan.eof() == scanError {
-		return nil, nil, scan.err
-	}
-	return data, nil, nil
-}
-
 // A SyntaxError is a description of a JSON syntax error.
 type SyntaxError struct {
 	msg    string // description of error
@@ -101,11 +72,6 @@ type scanner struct {
 	// Error that happened, if any.
 	err error
 
-	// 1-byte redo (see undo method)
-	redo      bool
-	redoCode  int
-	redoState func(*scanner, byte) int
-
 	// total bytes consumed, updated by decoder.Decode
 	bytes int64
 }
@@ -151,7 +117,6 @@ func (s *scanner) reset() {
 	s.step = stateBeginValue
 	s.parseState = s.parseState[0:0]
 	s.err = nil
-	s.redo = false
 	s.endTop = false
 }
 
@@ -184,7 +149,6 @@ func (s *scanner) pushParseState(p int) {
 func (s *scanner) popParseState() {
 	n := len(s.parseState) - 1
 	s.parseState = s.parseState[0:n]
-	s.redo = false
 	if n == 0 {
 		s.step = stateEndTop
 		s.endTop = true
@@ -607,22 +571,3 @@ func quoteChar(c byte) string {
 	s := strconv.Quote(string(c))
 	return "'" + s[1:len(s)-1] + "'"
 }
-
-// undo causes the scanner to return scanCode from the next state transition.
-// This gives callers a simple 1-byte undo mechanism.
-func (s *scanner) undo(scanCode int) {
-	if s.redo {
-		panic("json: invalid use of scanner")
-	}
-	s.redoCode = scanCode
-	s.redoState = s.step
-	s.step = stateRedo
-	s.redo = true
-}
-
-// stateRedo helps implement the scanner's 1-byte undo.
-func stateRedo(s *scanner, c byte) int {
-	s.redo = false
-	s.step = s.redoState
-	return s.redoCode
-}

src/encoding/json/scanner_test.go

@@ -200,43 +200,6 @@ func TestIndentErrors(t *testing.T) {
 	}
 }
 
-func TestNextValueBig(t *testing.T) {
-	initBig()
-	var scan scanner
-	item, rest, err := nextValue(jsonBig, &scan)
-	if err != nil {
-		t.Fatalf("nextValue: %s", err)
-	}
-	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
-		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
-	}
-	if len(rest) != 0 {
-		t.Errorf("invalid rest: %d", len(rest))
-	}
-
-	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
-	if err != nil {
-		t.Fatalf("nextValue extra: %s", err)
-	}
-	if len(item) != len(jsonBig) {
-		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
-	}
-	if string(rest) != "HELLO WORLD" {
-		t.Errorf("invalid rest: %d", len(rest))
-	}
-}
-
-var benchScan scanner
-
-func BenchmarkSkipValue(b *testing.B) {
-	initBig()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		nextValue(jsonBig, &benchScan)
-	}
-	b.SetBytes(int64(len(jsonBig)))
-}
-
 func diff(t *testing.T, a, b []byte) {
 	for i := 0; ; i++ {
 		if i >= len(a) || i >= len(b) || a[i] != b[i] {