regexp: hide one-pass code from exported API

Update #8112

Hide one-pass regexp API.

This means moving the code from regexp/syntax to regexp,
but it avoids being locked into the specific API chosen for
the implementation.

It also removes a slice field from the syntax.Inst, which
should avoid bloating the memory footprint of a non-one-pass
regexp unnecessarily.

LGTM=r
R=golang-codereviews, r
CC=golang-codereviews, iant
https://golang.org/cl/98610046

api/next.txt

@@ -330,12 +330,7 @@ pkg net/http, type Server struct, ConnState func(net.Conn, ConnState)
 pkg net/http, type Server struct, ErrorLog *log.Logger
 pkg net/http, type Transport struct, TLSHandshakeTimeout time.Duration
 pkg regexp/syntax, method (*Inst) MatchRunePos(int32) int
-pkg regexp/syntax, method (*Inst) OnePassNext(int32) uint32
-pkg regexp/syntax, method (*Prog) CompileOnePass() *Prog
-pkg regexp/syntax, method (*Prog) OnePassPrefix() (string, bool, uint32)
 pkg regexp/syntax, method (InstOp) String() string
-pkg regexp/syntax, type Inst struct, Next []uint32
-pkg regexp/syntax, var NotOnePass *Prog
 pkg runtime/debug, func SetPanicOnFault(bool) bool
 pkg runtime/debug, func WriteHeapDump(uintptr)
 pkg sync, method (*Pool) Get() interface{}

src/pkg/regexp/exec.go

@@ -37,7 +37,7 @@ type thread struct {
 type machine struct {
 	re       *Regexp      // corresponding Regexp
 	p        *syntax.Prog // compiled program
-	op       *syntax.Prog // compiled onepass program, or syntax.NotOnePass
+	op       *onePassProg // compiled onepass program, or notOnePass
 	q0, q1   queue        // two queues for runq, nextq
 	pool     []*thread    // pool of available threads
 	matched  bool         // whether a match was found
@@ -67,7 +67,7 @@ func (m *machine) newInputReader(r io.RuneReader) input {
 }
 
 // progMachine returns a new machine running the prog p.
-func progMachine(p, op *syntax.Prog) *machine {
+func progMachine(p *syntax.Prog, op *onePassProg) *machine {
 	m := &machine{p: p, op: op}
 	n := len(m.p.Inst)
 	m.q0 = queue{make([]uint32, n), make([]entry, 0, n)}
@@ -382,7 +382,7 @@ func (m *machine) onepass(i input, pos int) bool {
 			}
 		// peek at the input rune to see which branch of the Alt to take
 		case syntax.InstAlt, syntax.InstAltMatch:
-			pc = int(inst.OnePassNext(r))
+			pc = int(onePassNext(&inst, r))
 			continue
 		case syntax.InstFail:
 			return m.matched
@@ -429,7 +429,7 @@ func (re *Regexp) doExecute(r io.RuneReader, b []byte, s string, pos int, ncap i
 	} else {
 		i = m.newInputString(s)
 	}
-	if m.op != syntax.NotOnePass {
+	if m.op != notOnePass {
 		if !m.onepass(i, pos) {
 			re.put(m)
 			return nil

src/pkg/regexp/onepass_test.go

+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package regexp
+
+import (
+	"reflect"
+	"regexp/syntax"
+	"testing"
+)
+
+var runeMergeTests = []struct {
+	left, right, merged []rune
+	next                []uint32
+	leftPC, rightPC     uint32
+}{
+	{
+		// empty rhs
+		[]rune{69, 69},
+		[]rune{},
+		[]rune{69, 69},
+		[]uint32{1},
+		1, 2,
+	},
+	{
+		// identical runes, identical targets
+		[]rune{69, 69},
+		[]rune{69, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 1,
+	},
+	{
+		// identical runes, different targets
+		[]rune{69, 69},
+		[]rune{69, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// append right-first
+		[]rune{69, 69},
+		[]rune{71, 71},
+		[]rune{69, 69, 71, 71},
+		[]uint32{1, 2},
+		1, 2,
+	},
+	{
+		// append, left-first
+		[]rune{71, 71},
+		[]rune{69, 69},
+		[]rune{69, 69, 71, 71},
+		[]uint32{2, 1},
+		1, 2,
+	},
+	{
+		// successful interleave
+		[]rune{60, 60, 71, 71, 101, 101},
+		[]rune{69, 69, 88, 88},
+		[]rune{60, 60, 69, 69, 71, 71, 88, 88, 101, 101},
+		[]uint32{1, 2, 1, 2, 1},
+		1, 2,
+	},
+	{
+		// left surrounds right
+		[]rune{69, 74},
+		[]rune{71, 71},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// right surrounds left
+		[]rune{69, 74},
+		[]rune{68, 75},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap at interval begin
+		[]rune{69, 74},
+		[]rune{74, 75},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap ar interval end
+		[]rune{69, 74},
+		[]rune{65, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap from above
+		[]rune{69, 74},
+		[]rune{71, 74},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap from below
+		[]rune{69, 74},
+		[]rune{65, 71},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// out of order []rune
+		[]rune{69, 74, 60, 65},
+		[]rune{66, 67},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+}
+
+func TestMergeRuneSet(t *testing.T) {
+	for ix, test := range runeMergeTests {
+		merged, next := mergeRuneSets(&test.left, &test.right, test.leftPC, test.rightPC)
+		if !reflect.DeepEqual(merged, test.merged) {
+			t.Errorf("mergeRuneSet :%d (%v, %v) merged\n have\n%v\nwant\n%v", ix, test.left, test.right, merged, test.merged)
+		}
+		if !reflect.DeepEqual(next, test.next) {
+			t.Errorf("mergeRuneSet :%d(%v, %v) next\n have\n%v\nwant\n%v", ix, test.left, test.right, next, test.next)
+		}
+	}
+}
+
+const noStr = `!`
+
+var onePass = &onePassProg{}
+
+var onePassTests = []struct {
+	re      string
+	onePass *onePassProg
+	prog    string
+}{
+	{`^(?:a|(?:a*))$`, notOnePass, noStr},
+	{`^(?:(a)|(?:a*))$`, notOnePass, noStr},
+	{`^(?:(?:(?:.(?:$))?))$`, onePass, `a`},
+	{`^abcd$`, onePass, `abcd`},
+	{`^abcd$`, onePass, `abcde`},
+	{`^(?:(?:a{0,})*?)$`, onePass, `a`},
+	{`^(?:(?:a+)*)$`, onePass, ``},
+	{`^(?:(?:a|(?:aa)))$`, onePass, ``},
+	{`^(?:[^\s\S])$`, onePass, ``},
+	{`^(?:(?:a{3,4}){0,})$`, notOnePass, `aaaaaa`},
+	{`^(?:(?:a+)*)$`, onePass, `a`},
+	{`^(?:(?:(?:a*)+))$`, onePass, noStr},
+	{`^(?:(?:a+)*)$`, onePass, ``},
+	{`^[a-c]+$`, onePass, `abc`},
+	{`^[a-c]*$`, onePass, `abcdabc`},
+	{`^(?:a*)$`, onePass, `aaaaaaa`},
+	{`^(?:(?:aa)|a)$`, onePass, `a`},
+	{`^[a-c]*`, notOnePass, `abcdabc`},
+	{`^[a-c]*$`, onePass, `abc`},
+	{`^...$`, onePass, ``},
+	{`^(?:a|(?:aa))$`, onePass, `a`},
+	{`^[a-c]*`, notOnePass, `abcabc`},
+	{`^a((b))c$`, onePass, noStr},
+	{`^a.[l-nA-Cg-j]?e$`, onePass, noStr},
+	{`^a((b))$`, onePass, noStr},
+	{`^a(?:(b)|(c))c$`, onePass, noStr},
+	{`^a(?:(b*)|(c))c$`, notOnePass, noStr},
+	{`^a(?:b|c)$`, onePass, noStr},
+	{`^a(?:b?|c)$`, onePass, noStr},
+	{`^a(?:b?|c?)$`, notOnePass, noStr},
+	{`^a(?:b?|c+)$`, onePass, noStr},
+	{`^a(?:b+|(bc))d$`, notOnePass, noStr},
+	{`^a(?:bc)+$`, onePass, noStr},
+	{`^a(?:[bcd])+$`, onePass, noStr},
+	{`^a((?:[bcd])+)$`, onePass, noStr},
+	{`^a(:?b|c)*d$`, onePass, `abbbccbbcbbd"`},
+	{`^.bc(d|e)*$`, onePass, `abcddddddeeeededd`},
+	{`^(?:(?:aa)|.)$`, notOnePass, `a`},
+	{`^(?:(?:a{1,2}){1,2})$`, notOnePass, `aaaa`},
+}
+
+func TestCompileOnePass(t *testing.T) {
+	var (
+		p   *syntax.Prog
+		re  *syntax.Regexp
+		err error
+	)
+	for _, test := range onePassTests {
+		if re, err = syntax.Parse(test.re, syntax.Perl); err != nil {
+			t.Errorf("Parse(%q) got err:%s, want success", test.re, err)
+			continue
+		}
+		// needs to be done before compile...
+		re = re.Simplify()
+		if p, err = syntax.Compile(re); err != nil {
+			t.Errorf("Compile(%q) got err:%s, want success", test.re, err)
+			continue
+		}
+		onePass = compileOnePass(p)
+		if (onePass == notOnePass) != (test.onePass == notOnePass) {
+			t.Errorf("CompileOnePass(%q) got %v, expected %v", test.re, onePass, test.onePass)
+		}
+	}
+}

src/pkg/regexp/regexp.go

@@ -83,7 +83,7 @@ type Regexp struct {
 	// read-only after Compile
 	expr           string         // as passed to Compile
 	prog           *syntax.Prog   // compiled program
-	onepass        *syntax.Prog   // onpass program or nil
+	onepass        *onePassProg   // onpass program or nil
 	prefix         string         // required prefix in unanchored matches
 	prefixBytes    []byte         // prefix, as a []byte
 	prefixComplete bool           // prefix is the entire regexp
@@ -165,16 +165,16 @@ func compile(expr string, mode syntax.Flags, longest bool) (*Regexp, error) {
 	regexp := &Regexp{
 		expr:        expr,
 		prog:        prog,
-		onepass:     prog.CompileOnePass(),
+		onepass:     compileOnePass(prog),
 		numSubexp:   maxCap,
 		subexpNames: capNames,
 		cond:        prog.StartCond(),
 		longest:     longest,
 	}
-	if regexp.onepass == syntax.NotOnePass {
+	if regexp.onepass == notOnePass {
 		regexp.prefix, regexp.prefixComplete = prog.Prefix()
 	} else {
-		regexp.prefix, regexp.prefixComplete, regexp.prefixEnd = prog.OnePassPrefix()
+		regexp.prefix, regexp.prefixComplete, regexp.prefixEnd = onePassPrefix(prog)
 	}
 	if regexp.prefix != "" {
 		// TODO(rsc): Remove this allocation by adding

src/pkg/regexp/syntax/prog_test.go

@@ -4,10 +4,7 @@
 
 package syntax
 
-import (
-	"reflect"
-	"testing"
-)
+import "testing"
 
 var compileTests = []struct {
 	Regexp string
@@ -115,200 +112,3 @@ func BenchmarkEmptyOpContext(b *testing.B) {
 		EmptyOpContext(r1, -1)
 	}
 }
-
-var runeMergeTests = []struct {
-	left, right, merged []rune
-	next                []uint32
-	leftPC, rightPC     uint32
-}{
-	{
-		// empty rhs
-		[]rune{69, 69},
-		[]rune{},
-		[]rune{69, 69},
-		[]uint32{1},
-		1, 2,
-	},
-	{
-		// identical runes, identical targets
-		[]rune{69, 69},
-		[]rune{69, 69},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 1,
-	},
-	{
-		// identical runes, different targets
-		[]rune{69, 69},
-		[]rune{69, 69},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// append right-first
-		[]rune{69, 69},
-		[]rune{71, 71},
-		[]rune{69, 69, 71, 71},
-		[]uint32{1, 2},
-		1, 2,
-	},
-	{
-		// append, left-first
-		[]rune{71, 71},
-		[]rune{69, 69},
-		[]rune{69, 69, 71, 71},
-		[]uint32{2, 1},
-		1, 2,
-	},
-	{
-		// successful interleave
-		[]rune{60, 60, 71, 71, 101, 101},
-		[]rune{69, 69, 88, 88},
-		[]rune{60, 60, 69, 69, 71, 71, 88, 88, 101, 101},
-		[]uint32{1, 2, 1, 2, 1},
-		1, 2,
-	},
-	{
-		// left surrounds right
-		[]rune{69, 74},
-		[]rune{71, 71},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// right surrounds left
-		[]rune{69, 74},
-		[]rune{68, 75},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// overlap at interval begin
-		[]rune{69, 74},
-		[]rune{74, 75},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// overlap ar interval end
-		[]rune{69, 74},
-		[]rune{65, 69},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// overlap from above
-		[]rune{69, 74},
-		[]rune{71, 74},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// overlap from below
-		[]rune{69, 74},
-		[]rune{65, 71},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-	{
-		// out of order []rune
-		[]rune{69, 74, 60, 65},
-		[]rune{66, 67},
-		[]rune{},
-		[]uint32{mergeFailed},
-		1, 2,
-	},
-}
-
-func TestMergeRuneSet(t *testing.T) {
-	for ix, test := range runeMergeTests {
-		merged, next := mergeRuneSets(&test.left, &test.right, test.leftPC, test.rightPC)
-		if !reflect.DeepEqual(merged, test.merged) {
-			t.Errorf("mergeRuneSet :%d (%v, %v) merged\n have\n%v\nwant\n%v", ix, test.left, test.right, merged, test.merged)
-		}
-		if !reflect.DeepEqual(next, test.next) {
-			t.Errorf("mergeRuneSet :%d(%v, %v) next\n have\n%v\nwant\n%v", ix, test.left, test.right, next, test.next)
-		}
-	}
-}
-
-const noStr = `!`
-
-var onePass = &Prog{}
-
-var onePassTests = []struct {
-	re      string
-	onePass *Prog
-	prog    string
-}{
-	{`^(?:a|(?:a*))$`, NotOnePass, noStr},
-	{`^(?:(a)|(?:a*))$`, NotOnePass, noStr},
-	{`^(?:(?:(?:.(?:$))?))$`, onePass, `a`},
-	{`^abcd$`, onePass, `abcd`},
-	{`^abcd$`, onePass, `abcde`},
-	{`^(?:(?:a{0,})*?)$`, onePass, `a`},
-	{`^(?:(?:a+)*)$`, onePass, ``},
-	{`^(?:(?:a|(?:aa)))$`, onePass, ``},
-	{`^(?:[^\s\S])$`, onePass, ``},
-	{`^(?:(?:a{3,4}){0,})$`, NotOnePass, `aaaaaa`},
-	{`^(?:(?:a+)*)$`, onePass, `a`},
-	{`^(?:(?:(?:a*)+))$`, onePass, noStr},
-	{`^(?:(?:a+)*)$`, onePass, ``},
-	{`^[a-c]+$`, onePass, `abc`},
-	{`^[a-c]*$`, onePass, `abcdabc`},
-	{`^(?:a*)$`, onePass, `aaaaaaa`},
-	{`^(?:(?:aa)|a)$`, onePass, `a`},
-	{`^[a-c]*`, NotOnePass, `abcdabc`},
-	{`^[a-c]*$`, onePass, `abc`},
-	{`^...$`, onePass, ``},
-	{`^(?:a|(?:aa))$`, onePass, `a`},
-	{`^[a-c]*`, NotOnePass, `abcabc`},
-	{`^a((b))c$`, onePass, noStr},
-	{`^a.[l-nA-Cg-j]?e$`, onePass, noStr},
-	{`^a((b))$`, onePass, noStr},
-	{`^a(?:(b)|(c))c$`, onePass, noStr},
-	{`^a(?:(b*)|(c))c$`, NotOnePass, noStr},
-	{`^a(?:b|c)$`, onePass, noStr},
-	{`^a(?:b?|c)$`, onePass, noStr},
-	{`^a(?:b?|c?)$`, NotOnePass, noStr},
-	{`^a(?:b?|c+)$`, onePass, noStr},
-	{`^a(?:b+|(bc))d$`, NotOnePass, noStr},
-	{`^a(?:bc)+$`, onePass, noStr},
-	{`^a(?:[bcd])+$`, onePass, noStr},
-	{`^a((?:[bcd])+)$`, onePass, noStr},
-	{`^a(:?b|c)*d$`, onePass, `abbbccbbcbbd"`},
-	{`^.bc(d|e)*$`, onePass, `abcddddddeeeededd`},
-	{`^(?:(?:aa)|.)$`, NotOnePass, `a`},
-	{`^(?:(?:a{1,2}){1,2})$`, NotOnePass, `aaaa`},
-}
-
-func TestCompileOnePass(t *testing.T) {
-	var (
-		p   *Prog
-		re  *Regexp
-		err error
-	)
-	for _, test := range onePassTests {
-		if re, err = Parse(test.re, Perl); err != nil {
-			t.Errorf("Parse(%q) got err:%s, want success", test.re, err)
-			continue
-		}
-		// needs to be done before compile...
-		re = re.Simplify()
-		if p, err = Compile(re); err != nil {
-			t.Errorf("Compile(%q) got err:%s, want success", test.re, err)
-			continue
-		}
-		onePass = p.CompileOnePass()
-		if (onePass == NotOnePass) != (test.onePass == NotOnePass) {
-			t.Errorf("CompileOnePass(%q) got %v, expected %v", test.re, onePass, test.onePass)
-		}
-	}
-}