exp/html/atom: new package.

50% fewer mallocs in HTML tokenization, resulting in 25% fewer mallocs
in parsing go1.html.

Making the parser use integer comparisons instead of string comparisons
will be a follow-up CL, to be co-ordinated with Andy Balholm's work.

exp/html benchmarks before/after:

BenchmarkParser	     500	   4754294 ns/op	  16.44 MB/s
        parse_test.go:409: 500 iterations, 14651 mallocs per iteration
BenchmarkRawLevelTokenizer	    2000	    903481 ns/op	  86.51 MB/s
        token_test.go:678: 2000 iterations, 28 mallocs per iteration
BenchmarkLowLevelTokenizer	    2000	   1260485 ns/op	  62.01 MB/s
        token_test.go:678: 2000 iterations, 41 mallocs per iteration
BenchmarkHighLevelTokenizer	    1000	   2165964 ns/op	  36.09 MB/s
        token_test.go:678: 1000 iterations, 6616 mallocs per iteration

BenchmarkParser	     500	   4664912 ns/op	  16.76 MB/s
        parse_test.go:409: 500 iterations, 11266 mallocs per iteration
BenchmarkRawLevelTokenizer	    2000	    903065 ns/op	  86.55 MB/s
        token_test.go:678: 2000 iterations, 28 mallocs per iteration
BenchmarkLowLevelTokenizer	    2000	   1260032 ns/op	  62.03 MB/s
        token_test.go:678: 2000 iterations, 41 mallocs per iteration
BenchmarkHighLevelTokenizer	    1000	   2143356 ns/op	  36.47 MB/s
        token_test.go:678: 1000 iterations, 3231 mallocs per iteration

R=r, rsc, rogpeppe
CC=andybalholm, golang-dev
https://golang.org/cl/6255062

src/pkg/exp/html/atom/atom.go

+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package atom provides integer codes (also known as atoms) for a fixed set of
+// frequently occurring HTML strings: lower-case tag names and attribute keys
+// such as "p" and "id".
+//
+// Sharing an atom's string representation between all elements with the same
+// tag can result in fewer string allocations when tokenizing and parsing HTML.
+// Integer comparisons are also generally faster than string comparisons.
+//
+// An atom's particular code (such as atom.Div == 63) is not guaranteed to
+// stay the same between versions of this package. Neither is any ordering
+// guaranteed: whether atom.H1 < atom.H2 may also change. The codes are not
+// guaranteed to be dense. The only guarantees are that e.g. looking up "div"
+// will yield atom.Div, calling atom.Div.String will return "div", and
+// atom.Div != 0.
+package atom
+
+// Atom is an integer code for a string. The zero value maps to "".
+type Atom int
+
+// String returns the atom's string representation.
+func (a Atom) String() string {
+	if a <= 0 || a > max {
+		return ""
+	}
+	return table[a]
+}
+
+// Lookup returns the atom whose name is s. It returns zero if there is no
+// such atom.
+func Lookup(s []byte) Atom {
+	if len(s) == 0 {
+		return 0
+	}
+	if len(s) == 1 {
+		x := s[0]
+		if x < 'a' || x > 'z' {
+			return 0
+		}
+		return oneByteAtoms[x-'a']
+	}
+	// Binary search for the atom. Unlike sort.Search, this returns early on an exact match.
+	// TODO: this could be optimized further. For example, lo and hi could be initialized
+	// from s[0]. Separately, all the "onxxx" atoms could be moved into their own table.
+	lo, hi := Atom(1), 1+max
+	for lo < hi {
+		mid := (lo + hi) / 2
+		if cmp := compare(s, table[mid]); cmp == 0 {
+			return mid
+		} else if cmp > 0 {
+			lo = mid + 1
+		} else {
+			hi = mid
+		}
+	}
+	return 0
+}
+
+// String returns a string whose contents are equal to s. In that sense, it is
+// equivalent to string(s), but may be more efficient.
+func String(s []byte) string {
+	if a := Lookup(s); a != 0 {
+		return a.String()
+	}
+	return string(s)
+}
+
+// compare is like bytes.Compare, except that it takes one []byte argument and
+// one string argument, and returns negative/0/positive instead of -1/0/+1.
+func compare(s []byte, t string) int {
+	n := len(s)
+	if n > len(t) {
+		n = len(t)
+	}
+	for i, si := range s[:n] {
+		ti := t[i]
+		switch {
+		case si > ti:
+			return +1
+		case si < ti:
+			return -1
+		}
+	}
+	return len(s) - len(t)
+}

src/pkg/exp/html/atom/atom_test.go

+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package atom
+
+import (
+	"testing"
+)
+
+func TestHits(t *testing.T) {
+	for i, s := range table {
+		got := Lookup([]byte(s))
+		if got != Atom(i) {
+			t.Errorf("Lookup(%q): got %d, want %d", s, got, i)
+		}
+	}
+}
+
+func TestMisses(t *testing.T) {
+	testCases := []string{
+		"",
+		"\x00",
+		"\xff",
+		"A",
+		"DIV",
+		"Div",
+		"dIV",
+		"aa",
+		"a\x00",
+		"ab",
+		"abb",
+		"abbr0",
+		"abbr ",
+		" abbr",
+		" a",
+		"acceptcharset",
+		"acceptCharset",
+		"accept_charset",
+		"h0",
+		"h1h2",
+		"h7",
+		"onClick",
+		"λ",
+	}
+	for _, tc := range testCases {
+		got := Lookup([]byte(tc))
+		if got != 0 {
+			t.Errorf("Lookup(%q): got %d, want 0", tc, got)
+		}
+	}
+}

src/pkg/exp/html/atom/gen.go

+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This program generates table.go
+// Invoke as
+//
+//	go run gen.go |gofmt >table.go
+
+import (
+	"fmt"
+	"sort"
+)
+
+// identifier converts s to a Go exported identifier.
+// It converts "div" to "Div" and "accept-charset" to "AcceptCharset".
+func identifier(s string) string {
+	b := make([]byte, 0, len(s))
+	cap := true
+	for _, c := range s {
+		if c == '-' {
+			cap = true
+			continue
+		}
+		if cap && 'a' <= c && c <= 'z' {
+			c -= 'a' - 'A'
+		}
+		cap = false
+		b = append(b, byte(c))
+	}
+	return string(b)
+}
+
+func main() {
+	m := map[string]bool{
+		"": true,
+	}
+	for _, list := range [][]string{elements, attributes, eventHandlers, extra} {
+		for _, s := range list {
+			m[s] = true
+		}
+	}
+	atoms := make([]string, 0, len(m))
+	for s := range m {
+		atoms = append(atoms, s)
+	}
+	sort.Strings(atoms)
+
+	byInt := []string{}
+	byStr := map[string]int{}
+	ident := []string{}
+	for i, s := range atoms {
+		byInt = append(byInt, s)
+		byStr[s] = i
+		ident = append(ident, identifier(s))
+	}
+
+	fmt.Printf("package atom\n\nconst (\n")
+	for i, _ := range byInt {
+		if i == 0 {
+			continue
+		}
+		fmt.Printf("\t%s Atom = %d\n", ident[i], i)
+	}
+	fmt.Printf(")\n\n")
+	fmt.Printf("const max Atom = %d\n\n", len(byInt)-1)
+	fmt.Printf("var table = []string{\n")
+	for _, s := range byInt {
+		fmt.Printf("\t%q,\n", s)
+	}
+	fmt.Printf("}\n\n")
+	fmt.Printf("var oneByteAtoms = [26]Atom{\n")
+	for i := 'a'; i <= 'z'; i++ {
+		val := "0"
+		if x := byStr[string(i)]; x != 0 {
+			val = ident[x]
+		}
+		fmt.Printf("\t%s,\n", val)
+	}
+	fmt.Printf("}\n\n")
+}
+
+// The lists of element names and attribute keys were taken from
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/section-index.html
+// as of the "HTML Living Standard - Last Updated 30 May 2012" version.
+
+var elements = []string{
+	"a",
+	"abbr",
+	"address",
+	"area",
+	"article",
+	"aside",
+	"audio",
+	"b",
+	"base",
+	"bdi",
+	"bdo",
+	"blockquote",
+	"body",
+	"br",
+	"button",
+	"canvas",
+	"caption",
+	"cite",
+	"code",
+	"col",
+	"colgroup",
+	"command",
+	"data",
+	"datalist",
+	"dd",
+	"del",
+	"details",
+	"dfn",
+	"dialog",
+	"div",
+	"dl",
+	"dt",
+	"em",
+	"embed",
+	"fieldset",
+	"figcaption",
+	"figure",
+	"footer",
+	"form",
+	"h1",
+	"h2",
+	"h3",
+	"h4",
+	"h5",
+	"h6",
+	"head",
+	"header",
+	"hgroup",
+	"hr",
+	"html",
+	"i",
+	"iframe",
+	"img",
+	"input",
+	"ins",
+	"kbd",
+	"keygen",
+	"label",
+	"legend",
+	"li",
+	"link",
+	"map",
+	"mark",
+	"menu",
+	"meta",
+	"meter",
+	"nav",
+	"noscript",
+	"object",
+	"ol",
+	"optgroup",
+	"option",
+	"output",
+	"p",
+	"param",
+	"pre",
+	"progress",
+	"q",
+	"rp",
+	"rt",
+	"ruby",
+	"s",
+	"samp",
+	"script",
+	"section",
+	"select",
+	"small",
+	"source",
+	"span",
+	"strong",
+	"style",
+	"sub",
+	"summary",
+	"sup",
+	"table",
+	"tbody",
+	"td",
+	"textarea",
+	"tfoot",
+	"th",
+	"thead",
+	"time",
+	"title",
+	"tr",
+	"track",
+	"u",
+	"ul",
+	"var",
+	"video",
+	"wbr",
+}
+
+var attributes = []string{
+	"accept",
+	"accept-charset",
+	"accesskey",
+	"action",
+	"alt",
+	"async",
+	"autocomplete",
+	"autofocus",
+	"autoplay",
+	"border",
+	"challenge",
+	"charset",
+	"checked",
+	"cite",
+	"class",
+	"cols",
+	"colspan",
+	"command",
+	"content",
+	"contenteditable",
+	"contextmenu",
+	"controls",
+	"coords",
+	"crossorigin",
+	"data",
+	"datetime",
+	"default",
+	"defer",
+	"dir",
+	"dirname",
+	"disabled",
+	"download",
+	"draggable",
+	"dropzone",
+	"enctype",
+	"for",
+	"form",
+	"formaction",
+	"formenctype",
+	"formmethod",
+	"formnovalidate",
+	"formtarget",
+	"headers",
+	"height",
+	"hidden",
+	"high",
+	"href",
+	"hreflang",
+	"http-equiv",
+	"icon",
+	"id",
+	"inert",
+	"ismap",
+	"itemid",
+	"itemprop",
+	"itemref",
+	"itemscope",
+	"itemtype",
+	"keytype",
+	"kind",
+	"label",
+	"lang",
+	"list",
+	"loop",
+	"low",
+	"manifest",
+	"max",
+	"maxlength",
+	"media",
+	"mediagroup",
+	"method",
+	"min",
+	"multiple",
+	"muted",
+	"name",
+	"novalidate",
+	"open",
+	"optimum",
+	"pattern",
+	"ping",
+	"placeholder",
+	"poster",
+	"preload",
+	"radiogroup",
+	"readonly",
+	"rel",
+	"required",
+	"reversed",
+	"rows",
+	"rowspan",
+	"sandbox",
+	"spellcheck",
+	"scope",
+	"scoped",
+	"seamless",
+	"selected",
+	"shape",
+	"size",
+	"sizes",
+	"span",
+	"src",
+	"srcdoc",
+	"srclang",
+	"start",
+	"step",
+	"style",
+	"tabindex",
+	"target",
+	"title",
+	"translate",
+	"type",
+	"typemustmatch",
+	"usemap",
+	"value",
+	"width",
+	"wrap",
+}
+
+var eventHandlers = []string{
+	"onabort",
+	"onafterprint",
+	"onbeforeprint",
+	"onbeforeunload",
+	"onblur",
+	"oncancel",
+	"oncanplay",
+	"oncanplaythrough",
+	"onchange",
+	"onclick",
+	"onclose",
+	"oncontextmenu",
+	"oncuechange",
+	"ondblclick",
+	"ondrag",
+	"ondragend",
+	"ondragenter",
+	"ondragleave",
+	"ondragover",
+	"ondragstart",
+	"ondrop",
+	"ondurationchange",
+	"onemptied",
+	"onended",
+	"onerror",
+	"onfocus",
+	"onhashchange",
+	"oninput",
+	"oninvalid",
+	"onkeydown",
+	"onkeypress",
+	"onkeyup",
+	"onload",
+	"onloadeddata",
+	"onloadedmetadata",
+	"onloadstart",
+	"onmessage",
+	"onmousedown",
+	"onmousemove",
+	"onmouseout",
+	"onmouseover",
+	"onmouseup",
+	"onmousewheel",
+	"onoffline",
+	"ononline",
+	"onpagehide",
+	"onpageshow",
+	"onpause",
+	"onplay",
+	"onplaying",
+	"onpopstate",
+	"onprogress",
+	"onratechange",
+	"onreset",
+	"onresize",
+	"onscroll",
+	"onseeked",
+	"onseeking",
+	"onselect",
+	"onshow",
+	"onstalled",
+	"onstorage",
+	"onsubmit",
+	"onsuspend",
+	"ontimeupdate",
+	"onunload",
+	"onvolumechange",
+	"onwaiting",
+}
+
+// extra are ad-hoc values not covered by any of the lists above.
+var extra = []string{
+	"align",
+	"annotation",
+	"applet",
+	"center",
+	"color",
+	"font",
+	"frame",
+	"frameset",
+	"nobr",
+}

src/pkg/exp/html/token.go

@@ -6,6 +6,7 @@ package html
 
 import (
 	"bytes"
+	"exp/html/atom"
 	"io"
 	"strconv"
 	"strings"
@@ -791,13 +792,13 @@ func (z *Tokenizer) Token() Token {
 		for moreAttr {
 			var key, val []byte
 			key, val, moreAttr = z.TagAttr()
-			attr = append(attr, Attribute{"", string(key), string(val)})
+			attr = append(attr, Attribute{"", atom.String(key), string(val)})
 		}
-		t.Data = string(name)
+		t.Data = atom.String(name)
 		t.Attr = attr
 	case EndTagToken:
 		name, _ := z.TagName()
-		t.Data = string(name)
+		t.Data = atom.String(name)
 	}
 	return t
 }