net/http: import updated idna package and adjust request.go

Custom logic from request.go has been removed.

Created by running: “go run gen.go -core” from x/text
at fc7fa097411d30e6708badff276c4c164425590c.

Fixes golang/go#17268

Change-Id: Ie440d6ae30288352283d303e5126e5837f11bece
Reviewed-on: https://go-review.googlesource.com/37111
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

src/net/http/request.go

@@ -27,8 +27,6 @@ import (
 	"sync"
 
 	"golang_org/x/net/idna"
-	"golang_org/x/text/unicode/norm"
-	"golang_org/x/text/width"
 )
 
 const (
@@ -639,16 +637,19 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai
 type requestBodyReadError struct{ error }
 
 func idnaASCII(v string) (string, error) {
+	// TODO: Consider removing this check after verifying performance is okay.
+	// Right now punycode verification, length checks, context checks, and the
+	// permissable character tests are all omitted. It also prevents the ToASCII
+	// call from salvaging an invalid IDN, when possible. As a result it may be
+	// possible to have two IDNs that appear identical to the user where the
+	// ASCII-only version causes an error downstream whereas the non-ASCII
+	// version does not.
+	// Note that for correct ASCII IDNs ToASCII will only do considerably more
+	// work, but it will not cause an allocation.
 	if isASCII(v) {
 		return v, nil
 	}
-	// The idna package doesn't do everything from
-	// https://tools.ietf.org/html/rfc5895 so we do it here.
-	// TODO(bradfitz): should the idna package do this instead?
-	v = strings.ToLower(v)
-	v = width.Fold.String(v)
-	v = norm.NFC.String(v)
-	return idna.ToASCII(v)
+	return idna.Lookup.ToASCII(v)
 }
 
 // cleanHost cleans up the host sent in request's Host header.

src/vendor/golang_org/x/net/idna/idna_test.go

-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package idna
-
-import (
-	"testing"
-)
-
-var idnaTestCases = [...]struct {
-	ascii, unicode string
-}{
-	// Labels.
-	{"books", "books"},
-	{"xn--bcher-kva", "bücher"},
-
-	// Domains.
-	{"foo--xn--bar.org", "foo--xn--bar.org"},
-	{"golang.org", "golang.org"},
-	{"example.xn--p1ai", "example.рф"},
-	{"xn--czrw28b.tw", "商業.tw"},
-	{"www.xn--mller-kva.de", "www.müller.de"},
-}
-
-func TestIDNA(t *testing.T) {
-	for _, tc := range idnaTestCases {
-		if a, err := ToASCII(tc.unicode); err != nil {
-			t.Errorf("ToASCII(%q): %v", tc.unicode, err)
-		} else if a != tc.ascii {
-			t.Errorf("ToASCII(%q): got %q, want %q", tc.unicode, a, tc.ascii)
-		}
-
-		if u, err := ToUnicode(tc.ascii); err != nil {
-			t.Errorf("ToUnicode(%q): %v", tc.ascii, err)
-		} else if u != tc.unicode {
-			t.Errorf("ToUnicode(%q): got %q, want %q", tc.ascii, u, tc.unicode)
-		}
-	}
-}
-
-// TODO(nigeltao): test errors, once we've specified when ToASCII and ToUnicode
-// return errors.

src/vendor/golang_org/x/net/idna/punycode.go

-// Copyright 2012 The Go Authors. All rights reserved.
+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -7,7 +9,6 @@ package idna
 // This file implements the Punycode algorithm from RFC 3492.
 
 import (
-	"fmt"
 	"math"
 	"strings"
 	"unicode/utf8"
@@ -27,6 +28,8 @@ const (
 	tmin        int32 = 1
 )
 
+func punyError(s string) error { return &labelError{s, "A3"} }
+
 // decode decodes a string as specified in section 6.2.
 func decode(encoded string) (string, error) {
 	if encoded == "" {
@@ -34,7 +37,7 @@ func decode(encoded string) (string, error) {
 	}
 	pos := 1 + strings.LastIndex(encoded, "-")
 	if pos == 1 {
-		return "", fmt.Errorf("idna: invalid label %q", encoded)
+		return "", punyError(encoded)
 	}
 	if pos == len(encoded) {
 		return encoded[:len(encoded)-1], nil
@@ -50,16 +53,16 @@ func decode(encoded string) (string, error) {
 		oldI, w := i, int32(1)
 		for k := base; ; k += base {
 			if pos == len(encoded) {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			digit, ok := decodeDigit(encoded[pos])
 			if !ok {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			pos++
 			i += digit * w
 			if i < 0 {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 			t := k - bias
 			if t < tmin {
@@ -72,7 +75,7 @@ func decode(encoded string) (string, error) {
 			}
 			w *= base - t
 			if w >= math.MaxInt32/base {
-				return "", fmt.Errorf("idna: invalid label %q", encoded)
+				return "", punyError(encoded)
 			}
 		}
 		x := int32(len(output) + 1)
@@ -80,7 +83,7 @@ func decode(encoded string) (string, error) {
 		n += i / x
 		i %= x
 		if n > utf8.MaxRune || len(output) >= 1024 {
-			return "", fmt.Errorf("idna: invalid label %q", encoded)
+			return "", punyError(encoded)
 		}
 		output = append(output, 0)
 		copy(output[i+1:], output[i:])
@@ -121,14 +124,14 @@ func encode(prefix, s string) (string, error) {
 		}
 		delta += (m - n) * (h + 1)
 		if delta < 0 {
-			return "", fmt.Errorf("idna: invalid label %q", s)
+			return "", punyError(s)
 		}
 		n = m
 		for _, r := range s {
 			if r < n {
 				delta++
 				if delta < 0 {
-					return "", fmt.Errorf("idna: invalid label %q", s)
+					return "", punyError(s)
 				}
 				continue
 			}

src/vendor/golang_org/x/net/idna/trie.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package idna
+
+// appendMapping appends the mapping for the respective rune. isMapped must be
+// true. A mapping is a categorization of a rune as defined in UTS #46.
+func (c info) appendMapping(b []byte, s string) []byte {
+	index := int(c >> indexShift)
+	if c&xorBit == 0 {
+		s := mappings[index:]
+		return append(b, s[1:s[0]+1]...)
+	}
+	b = append(b, s...)
+	if c&inlineXOR == inlineXOR {
+		// TODO: support and handle two-byte inline masks
+		b[len(b)-1] ^= byte(index)
+	} else {
+		for p := len(b) - int(xorData[index]); p < len(b); p++ {
+			index++
+			b[p] ^= xorData[index]
+		}
+	}
+	return b
+}
+
+// Sparse block handling code.
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var idnaSparse = sparseBlocks{
+	values: idnaSparseValues[:],
+	offset: idnaSparseOffset[:],
+}
+
+// Don't use newIdnaTrie to avoid unconditional linking in of the table.
+var trie = &idnaTrie{}
+
+// lookup determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

src/vendor/golang_org/x/net/idna/trieval.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT.
+
+package idna
+
+// This file contains definitions for interpreting the trie value of the idna
+// trie generated by "go run gen*.go". It is shared by both the generator
+// program and the resultant package. Sharing is achieved by the generator
+// copying gen_trieval.go to trieval.go and changing what's above this comment.
+
+// info holds information from the IDNA mapping table for a single rune. It is
+// the value returned by a trie lookup. In most cases, all information fits in
+// a 16-bit value. For mappings, this value may contain an index into a slice
+// with the mapped string. Such mappings can consist of the actual mapped value
+// or an XOR pattern to be applied to the bytes of the UTF8 encoding of the
+// input rune. This technique is used by the cases packages and reduces the
+// table size significantly.
+//
+// The per-rune values have the following format:
+//
+//   if mapped {
+//     if inlinedXOR {
+//       15..13 inline XOR marker
+//       12..11 unused
+//       10..3  inline XOR mask
+//     } else {
+//       15..3  index into xor or mapping table
+//     }
+//   } else {
+//       15..13 unused
+//           12 modifier (including virama)
+//           11 virama modifier
+//       10..8  joining type
+//        7..3  category type
+//   }
+//      2  use xor pattern
+//   1..0  mapped category
+//
+// See the definitions below for a more detailed description of the various
+// bits.
+type info uint16
+
+const (
+	catSmallMask = 0x3
+	catBigMask   = 0xF8
+	indexShift   = 3
+	xorBit       = 0x4    // interpret the index as an xor pattern
+	inlineXOR    = 0xE000 // These bits are set if the XOR pattern is inlined.
+
+	joinShift = 8
+	joinMask  = 0x07
+
+	viramaModifier = 0x0800
+	modifier       = 0x1000
+)
+
+// A category corresponds to a category defined in the IDNA mapping table.
+type category uint16
+
+const (
+	unknown              category = 0 // not defined currently in unicode.
+	mapped               category = 1
+	disallowedSTD3Mapped category = 2
+	deviation            category = 3
+)
+
+const (
+	valid               category = 0x08
+	validNV8            category = 0x18
+	validXV8            category = 0x28
+	disallowed          category = 0x40
+	disallowedSTD3Valid category = 0x80
+	ignored             category = 0xC0
+)
+
+// join types and additional rune information
+const (
+	joiningL = (iota + 1)
+	joiningD
+	joiningT
+	joiningR
+
+	//the following types are derived during processing
+	joinZWJ
+	joinZWNJ
+	joinVirama
+	numJoinTypes
+)
+
+func (c info) isMapped() bool {
+	return c&0x3 != 0
+}
+
+func (c info) category() category {
+	small := c & catSmallMask
+	if small != 0 {
+		return category(small)
+	}
+	return category(c & catBigMask)
+}
+
+func (c info) joinType() info {
+	if c.isMapped() {
+		return 0
+	}
+	return (c >> joinShift) & joinMask
+}
+
+func (c info) isModifier() bool {
+	return c&(modifier|catSmallMask) == modifier
+}
+
+func (c info) isViramaModifier() bool {
+	return c&(viramaModifier|catSmallMask) == viramaModifier
+}

src/vendor/golang_org/x/text/secure/bidirule/bidirule.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bidirule implements the Bidi Rule defined by RFC 5893.
+//
+// This package is under development. The API may change without notice and
+// without preserving backward compatibility.
+package bidirule
+
+import (
+	"errors"
+	"unicode/utf8"
+
+	"golang_org/x/text/transform"
+	"golang_org/x/text/unicode/bidi"
+)
+
+// This file contains an implementation of RFC 5893: Right-to-Left Scripts for
+// Internationalized Domain Names for Applications (IDNA)
+//
+// A label is an individual component of a domain name.  Labels are usually
+// shown separated by dots; for example, the domain name "www.example.com" is
+// composed of three labels: "www", "example", and "com".
+//
+// An RTL label is a label that contains at least one character of class R, AL,
+// or AN. An LTR label is any label that is not an RTL label.
+//
+// A "Bidi domain name" is a domain name that contains at least one RTL label.
+//
+//  The following guarantees can be made based on the above:
+//
+//  o  In a domain name consisting of only labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied.  Note that even LTR
+//     labels and pure ASCII labels have to be tested.
+//
+//  o  In a domain name consisting of only LDH labels (as defined in the
+//     Definitions document [RFC5890]) and labels that satisfy the rule,
+//     the requirements of Section 3 are satisfied as long as a label
+//     that starts with an ASCII digit does not come after a
+//     right-to-left label.
+//
+//  No guarantee is given for other combinations.
+
+// ErrInvalid indicates a label is invalid according to the Bidi Rule.
+var ErrInvalid = errors.New("bidirule: failed Bidi Rule")
+
+type ruleState uint8
+
+const (
+	ruleInitial ruleState = iota
+	ruleLTR
+	ruleLTRFinal
+	ruleRTL
+	ruleRTLFinal
+	ruleInvalid
+)
+
+type ruleTransition struct {
+	next ruleState
+	mask uint16
+}
+
+var transitions = [...][2]ruleTransition{
+	// [2.1] The first character must be a character with Bidi property L, R, or
+	// AL. If it has the R or AL property, it is an RTL label; if it has the L
+	// property, it is an LTR label.
+	ruleInitial: {
+		{ruleLTRFinal, 1 << bidi.L},
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL},
+	},
+	ruleRTL: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3]
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleRTLFinal: {
+		// [2.3] In an RTL label, the end of the label must be a character with
+		// Bidi property R, AL, EN, or AN, followed by zero or more characters
+		// with Bidi property NSM.
+		{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN | 1<<bidi.NSM},
+
+		// [2.2] In an RTL label, only characters with the Bidi properties R,
+		// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.3] and NSM.
+		{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleLTR: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
+	},
+	ruleLTRFinal: {
+		// [2.6] In an LTR label, the end of the label must be a character with
+		// Bidi property L or EN, followed by zero or more characters with Bidi
+		// property NSM.
+		{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN | 1<<bidi.NSM},
+
+		// [2.5] In an LTR label, only characters with the Bidi properties L,
+		// EN, ES, CS, ET, ON, BN, or NSM are allowed.
+		// We exclude the entries from [2.6].
+		{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
+	},
+	ruleInvalid: {
+		{ruleInvalid, 0},
+		{ruleInvalid, 0},
+	},
+}
+
+// [2.4] In an RTL label, if an EN is present, no AN may be present, and
+// vice versa.
+const exclusiveRTL = uint16(1<<bidi.EN | 1<<bidi.AN)
+
+// From RFC 5893
+// An RTL label is a label that contains at least one character of type
+// R, AL, or AN.
+//
+// An LTR label is any label that is not an RTL label.
+
+// Direction reports the direction of the given label as defined by RFC 5893.
+// The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func Direction(b []byte) bidi.Direction {
+	for i := 0; i < len(b); {
+		e, sz := bidi.Lookup(b[i:])
+		if sz == 0 {
+			i++
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// DirectionString reports the direction of the given label as defined by RFC
+// 5893. The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func DirectionString(s string) bidi.Direction {
+	for i := 0; i < len(s); {
+		e, sz := bidi.LookupString(s[i:])
+		if sz == 0 {
+			i++
+		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
+	}
+	return bidi.LeftToRight
+}
+
+// Valid reports whether b conforms to the BiDi rule.
+func Valid(b []byte) bool {
+	var t Transformer
+	if n, ok := t.advance(b); !ok || n < len(b) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// ValidString reports whether s conforms to the BiDi rule.
+func ValidString(s string) bool {
+	var t Transformer
+	if n, ok := t.advanceString(s); !ok || n < len(s) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// New returns a Transformer that verifies that input adheres to the Bidi Rule.
+func New() *Transformer {
+	return &Transformer{}
+}
+
+// Transformer implements transform.Transform.
+type Transformer struct {
+	state  ruleState
+	hasRTL bool
+	seen   uint16
+}
+
+// A rule can only be violated for "Bidi Domain names", meaning if one of the
+// following categories has been observed.
+func (t *Transformer) isRTL() bool {
+	const isRTL = 1<<bidi.R | 1<<bidi.AL | 1<<bidi.AN
+	return t.seen&isRTL != 0
+}
+
+func (t *Transformer) isFinal() bool {
+	if !t.isRTL() {
+		return true
+	}
+	return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
+}
+
+// Reset implements transform.Transformer.
+func (t *Transformer) Reset() { *t = Transformer{} }
+
+// Transform implements transform.Transformer. This Transformer has state and
+// needs to be reset between uses.
+func (t *Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if len(dst) < len(src) {
+		src = src[:len(dst)]
+		atEOF = false
+		err = transform.ErrShortDst
+	}
+	n, err1 := t.Span(src, atEOF)
+	copy(dst, src[:n])
+	if err == nil || err1 != nil && err1 != transform.ErrShortSrc {
+		err = err1
+	}
+	return n, n, err
+}
+
+// Span returns the first n bytes of src that conform to the Bidi rule.
+func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
+	if t.state == ruleInvalid && t.isRTL() {
+		return 0, ErrInvalid
+	}
+	n, ok := t.advance(src)
+	switch {
+	case !ok:
+		err = ErrInvalid
+	case n < len(src):
+		if !atEOF {
+			err = transform.ErrShortSrc
+			break
+		}
+		err = ErrInvalid
+	case !t.isFinal():
+		err = ErrInvalid
+	}
+	return n, err
+}
+
+// Precomputing the ASCII values decreases running time for the ASCII fast path
+// by about 30%.
+var asciiTable [128]bidi.Properties
+
+func init() {
+	for i := range asciiTable {
+		p, _ := bidi.LookupRune(rune(i))
+		asciiTable[i] = p
+	}
+}
+
+func (t *Transformer) advance(s []byte) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.Lookup(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					// We always consider invalid UTF-8 to be invalid, even if
+					// the string has not yet been determined to be RTL.
+					// TODO: is this correct?
+					return n, false
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass would result in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}
+
+func (t *Transformer) advanceString(s string) (n int, ok bool) {
+	var e bidi.Properties
+	var sz int
+	for n < len(s) {
+		if s[n] < utf8.RuneSelf {
+			e, sz = asciiTable[s[n]], 1
+		} else {
+			e, sz = bidi.LookupString(s[n:])
+			if sz <= 1 {
+				if sz == 1 {
+					return n, false // invalid UTF-8
+				}
+				return n, true // incomplete UTF-8 encoding
+			}
+		}
+		// TODO: using CompactClass results in noticeable speedup.
+		// See unicode/bidi/prop.go:Properties.CompactClass.
+		c := uint16(1 << e.Class())
+		t.seen |= c
+		if t.seen&exclusiveRTL == exclusiveRTL {
+			t.state = ruleInvalid
+			return n, false
+		}
+		switch tr := transitions[t.state]; {
+		case tr[0].mask&c != 0:
+			t.state = tr[0].next
+		case tr[1].mask&c != 0:
+			t.state = tr[1].next
+		default:
+			t.state = ruleInvalid
+			if t.isRTL() {
+				return n, false
+			}
+		}
+		n += sz
+	}
+	return n, true
+}

src/vendor/golang_org/x/text/secure/doc.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// secure is a repository of text security related packages.
+package secure // import "golang_org/x/text/secure"

src/vendor/golang_org/x/text/transform/examples_test.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package transform_test
+
+import (
+	"fmt"
+	"unicode"
+
+	"golang_org/x/text/transform"
+	"golang_org/x/text/unicode/norm"
+)
+
+func ExampleRemoveFunc() {
+	input := []byte(`tschüß; до свидания`)
+
+	b := make([]byte, len(input))
+
+	t := transform.RemoveFunc(unicode.IsSpace)
+	n, _, _ := t.Transform(b, input, true)
+	fmt.Println(string(b[:n]))
+
+	t = transform.RemoveFunc(func(r rune) bool {
+		return !unicode.Is(unicode.Latin, r)
+	})
+	n, _, _ = t.Transform(b, input, true)
+	fmt.Println(string(b[:n]))
+
+	n, _, _ = t.Transform(b, norm.NFD.Bytes(input), true)
+	fmt.Println(string(b[:n]))
+
+	// Output:
+	// tschüß;досвидания
+	// tschüß
+	// tschuß
+}

src/vendor/golang_org/x/text/transform/transform.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -6,7 +8,7 @@
 // bytes passing through as well as various transformations. Example
 // transformations provided by other packages include normalization and
 // conversion between character sets.
-package transform // import "golang.org/x/text/transform"
+package transform // import "golang_org/x/text/transform"
 
 import (
 	"bytes"

src/vendor/golang_org/x/text/unicode/bidi/bidi.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bidi contains functionality for bidirectional text support.
+//
+// See http://www.unicode.org/reports/tr9.
+//
+// NOTE: UNDER CONSTRUCTION. This API may change in backwards incompatible ways
+// and without notice.
+package bidi // import "golang_org/x/text/unicode/bidi"
+
+// TODO:
+// The following functionality would not be hard to implement, but hinges on
+// the definition of a Segmenter interface. For now this is up to the user.
+// - Iterate over paragraphs
+// - Segmenter to iterate over runs directly from a given text.
+// Also:
+// - Transformer for reordering?
+// - Transformer (validator, really) for Bidi Rule.
+
+// This API tries to avoid dealing with embedding levels for now. Under the hood
+// these will be computed, but the question is to which extent the user should
+// know they exist. We should at some point allow the user to specify an
+// embedding hierarchy, though.
+
+// A Direction indicates the overall flow of text.
+type Direction int
+
+const (
+	// LeftToRight indicates the text contains no right-to-left characters and
+	// that either there are some left-to-right characters or the option
+	// DefaultDirection(LeftToRight) was passed.
+	LeftToRight Direction = iota
+
+	// RightToLeft indicates the text contains no left-to-right characters and
+	// that either there are some right-to-left characters or the option
+	// DefaultDirection(RightToLeft) was passed.
+	RightToLeft
+
+	// Mixed indicates text contains both left-to-right and right-to-left
+	// characters.
+	Mixed
+
+	// Neutral means that text contains no left-to-right and right-to-left
+	// characters and that no default direction has been set.
+	Neutral
+)
+
+type options struct{}
+
+// An Option is an option for Bidi processing.
+type Option func(*options)
+
+// ICU allows the user to define embedding levels. This may be used, for example,
+// to use hierarchical structure of markup languages to define embeddings.
+// The following option may be a way to expose this functionality in this API.
+// // LevelFunc sets a function that associates nesting levels with the given text.
+// // The levels function will be called with monotonically increasing values for p.
+// func LevelFunc(levels func(p int) int) Option {
+// 	panic("unimplemented")
+// }
+
+// DefaultDirection sets the default direction for a Paragraph. The direction is
+// overridden if the text contains directional characters.
+func DefaultDirection(d Direction) Option {
+	panic("unimplemented")
+}
+
+// A Paragraph holds a single Paragraph for Bidi processing.
+type Paragraph struct {
+	// buffers
+}
+
+// SetBytes configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// SetString configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// IsLeftToRight reports whether the principle direction of rendering for this
+// paragraphs is left-to-right. If this returns false, the principle direction
+// of rendering is right-to-left.
+func (p *Paragraph) IsLeftToRight() bool {
+	panic("unimplemented")
+}
+
+// Direction returns the direction of the text of this paragraph.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (p *Paragraph) Direction() Direction {
+	panic("unimplemented")
+}
+
+// RunAt reports the Run at the given position of the input text.
+//
+// This method can be used for computing line breaks on paragraphs.
+func (p *Paragraph) RunAt(pos int) Run {
+	panic("unimplemented")
+}
+
+// Order computes the visual ordering of all the runs in a Paragraph.
+func (p *Paragraph) Order() (Ordering, error) {
+	panic("unimplemented")
+}
+
+// Line computes the visual ordering of runs for a single line starting and
+// ending at the given positions in the original text.
+func (p *Paragraph) Line(start, end int) (Ordering, error) {
+	panic("unimplemented")
+}
+
+// An Ordering holds the computed visual order of runs of a Paragraph. Calling
+// SetBytes or SetString on the originating Paragraph invalidates an Ordering.
+// The methods of an Ordering should only be called by one goroutine at a time.
+type Ordering struct{}
+
+// Direction reports the directionality of the runs.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (o *Ordering) Direction() Direction {
+	panic("unimplemented")
+}
+
+// NumRuns returns the number of runs.
+func (o *Ordering) NumRuns() int {
+	panic("unimplemented")
+}
+
+// Run returns the ith run within the ordering.
+func (o *Ordering) Run(i int) Run {
+	panic("unimplemented")
+}
+
+// TODO: perhaps with options.
+// // Reorder creates a reader that reads the runes in visual order per character.
+// // Modifiers remain after the runes they modify.
+// func (l *Runs) Reorder() io.Reader {
+// 	panic("unimplemented")
+// }
+
+// A Run is a continuous sequence of characters of a single direction.
+type Run struct {
+}
+
+// String returns the text of the run in its original order.
+func (r *Run) String() string {
+	panic("unimplemented")
+}
+
+// Bytes returns the text of the run in its original order.
+func (r *Run) Bytes() []byte {
+	panic("unimplemented")
+}
+
+// TODO: methods for
+// - Display order
+// - headers and footers
+// - bracket replacement.
+
+// Direction reports the direction of the run.
+func (r *Run) Direction() Direction {
+	panic("unimplemented")
+}
+
+// Position of the Run within the text passed to SetBytes or SetString of the
+// originating Paragraph value.
+func (r *Run) Pos() (start, end int) {
+	panic("unimplemented")
+}
+
+// AppendReverse reverses the order of characters of in, appends them to out,
+// and returns the result. Modifiers will still follow the runes they modify.
+// Brackets are replaced with their counterparts.
+func AppendReverse(out, in []byte) []byte {
+	panic("unimplemented")
+}
+
+// ReverseString reverses the order of characters in s and returns a new string.
+// Modifiers will still follow the runes they modify. Brackets are replaced with
+// their counterparts.
+func ReverseString(s string) string {
+	panic("unimplemented")
+}

src/vendor/golang_org/x/text/unicode/bidi/example_test.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// +build ignore
+
+package bidi_test
+
+import (
+	"fmt"
+	"log"
+
+	"golang_org/x/text/bidi"
+)
+
+func foo() {
+	var sa StringAttributes
+	var p Paragraph
+	n, _ := p.SetString(s)
+	for i, o := 0, p.Ordering(); i < o.NumRuns(); i++ {
+		b := o.Run(i).Bytes()
+
+		start, end := o.Run(i).Pos()
+		for p := start; p < end; {
+			style, n := sa.StyleAt(start)
+			render()
+			p += n
+		}
+
+	}
+}
+
+type style int
+
+const (
+	styleNormal   = 0
+	styleSelected = 1 << (iota - 1)
+	styleBold
+	styleItalics
+)
+
+type styleRun struct {
+	end   int
+	style style
+}
+
+func getTextWidth(text string, styleRuns []styleRun) int {
+	// simplistic way to compute the width
+	return len([]rune(text))
+}
+
+// set limit and StyleRun limit for a line
+// from text[start] and from styleRuns[styleRunStart]
+// using Bidi.getLogicalRun(...)
+// returns line width
+func getLineBreak(p *bidi.Paragraph, start int, styles []styleRun) (n int) {
+	// dummy return
+	return 0
+}
+
+// render runs on a line sequentially, always from left to right
+
+// prepare rendering a new line
+func startLine(d bidi.Direction, lineWidth int) {
+	fmt.Println()
+}
+
+// render a run of text and advance to the right by the run width
+// the text[start..limit-1] is always in logical order
+func renderRun(text string, d bidi.Direction, styl style) {
+}
+
+// We could compute a cross-product
+// from the style runs with the directional runs
+// and then reorder it.
+// Instead, here we iterate over each run type
+// and render the intersections -
+// with shortcuts in simple (and common) cases.
+// renderParagraph() is the main function.
+
+// render a directional run with
+// (possibly) multiple style runs intersecting with it
+func renderDirectionalRun(text string, offset int, d bidi.Direction, styles []styleRun) {
+	start, end := offset, len(text)+offset
+	// iterate over style runs
+	if run.Direction() == bidi.LeftToRight {
+		styleEnd := 0
+		for _, sr := range styles {
+			styleEnd = styleRuns[i].end
+			if start < styleEnd {
+				if styleEnd > end {
+					styleEnd = end
+				}
+				renderRun(text[start-offset:styleEnd-offset], run.Direction(), styles[i].style)
+				if styleEnd == end {
+					break
+				}
+				start = styleEnd
+			}
+		}
+	} else {
+		styleStart := 0
+		for i := len(styles) - 1; i >= 0; i-- {
+			if i > 0 {
+				styleStart = styles[i-1].end
+			} else {
+				styleStart = 0
+			}
+			if end >= styleStart {
+				if styleStart < start {
+					styleStart = start
+				}
+				renderRun(text[styleStart-offset:end-offset], run.Direction(), styles[i].style)
+				if styleStart == start {
+					break
+				}
+				end = styleStart
+			}
+		}
+	}
+}
+
+// the line object represents text[start..limit-1]
+func renderLine(line *bidi.Runs, text string, offset int, styles []styleRun) {
+	if dir := line.Direction(); dir != bidi.Mixed {
+		if len(styles) == 1 {
+			renderRun(text, dir, styles[0].style)
+		} else {
+			for i := 0; i < line.NumRuns(); i++ {
+				renderDirectionalRun(text, offset, dir, styles)
+			}
+		}
+	} else {
+		// iterate over both directional and style runs
+		for i := 0; i < line.Len(); i++ {
+			run := line.Run(i)
+			start, _ := run.Pos()
+			renderDirectionalRun(text[start-offset:], start, run.Direction(), styles)
+		}
+	}
+}
+
+func renderParagraph(text string, d bidi.Direction, styles []styleRun, int lineWidth) {
+	var p bidi.Paragraph
+	if err := p.SetString(text, bidi.DefaultDirection(d)); err != nil {
+		log.Fatal(err)
+	}
+
+	if len(styles) == 0 {
+		styles = append(styles, []styleRun{len(text), styleNormal})
+	}
+
+	if width := getTextWidth(text, styles); width <= lineWidth {
+		// everything fits onto one line
+
+		runs, err := p.Runs()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// prepare rendering a new line from either left or right
+		startLine(p.Direction(), width)
+		renderLine(&runs, text, styles)
+	} else {
+		// we need to render several lines
+
+		for start, end := 0, 0; start < len(text); start = end {
+			for start >= styles[0].end {
+				styles = styles[1:]
+			}
+			end = getLineBreak(p, start, styles[startStyles:])
+
+			runs, err := p.Line(start, end)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			startLine(p.Direction(), end-start)
+			renderLine(&runs, text[start:end], styles[startStyles:])
+		}
+	}
+}
+
+func main() {
+	renderParagraph("Some Latin text...", bidi.LeftToRight, nil, 80)
+	renderParagraph("Some Hebrew text...", bidi.RightToLeft, nil, 60)
+}

src/vendor/golang_org/x/text/unicode/bidi/prop.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import "unicode/utf8"
+
+// Properties provides access to BiDi properties of runes.
+type Properties struct {
+	entry uint8
+	last  uint8
+}
+
+var trie = newBidiTrie(0)
+
+// TODO: using this for bidirule reduces the running time by about 5%. Consider
+// if this is worth exposing or if we can find a way to speed up the Class
+// method.
+//
+// // CompactClass is like Class, but maps all of the BiDi control classes
+// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control.
+// func (p Properties) CompactClass() Class {
+// 	return Class(p.entry & 0x0F)
+// }
+
+// Class returns the Bidi class for p.
+func (p Properties) Class() Class {
+	c := Class(p.entry & 0x0F)
+	if c == Control {
+		c = controlByteToClass[p.last&0xF]
+	}
+	return c
+}
+
+// IsBracket reports whether the rune is a bracket.
+func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 }
+
+// IsOpeningBracket reports whether the rune is an opening bracket.
+// IsBracket must return true.
+func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 }
+
+// TODO: find a better API and expose.
+func (p Properties) reverseBracket(r rune) rune {
+	return xorMasks[p.entry>>xorMaskShift] ^ r
+}
+
+var controlByteToClass = [16]Class{
+	0xD: LRO, // U+202D LeftToRightOverride,
+	0xE: RLO, // U+202E RightToLeftOverride,
+	0xA: LRE, // U+202A LeftToRightEmbedding,
+	0xB: RLE, // U+202B RightToLeftEmbedding,
+	0xC: PDF, // U+202C PopDirectionalFormat,
+	0x6: LRI, // U+2066 LeftToRightIsolate,
+	0x7: RLI, // U+2067 RightToLeftIsolate,
+	0x8: FSI, // U+2068 FirstStrongIsolate,
+	0x9: PDI, // U+2069 PopDirectionalIsolate,
+}
+
+// LookupRune returns properties for r.
+func LookupRune(r rune) (p Properties, size int) {
+	var buf [4]byte
+	n := utf8.EncodeRune(buf[:], r)
+	return Lookup(buf[:n])
+}
+
+// TODO: these lookup methods are based on the generated trie code. The returned
+// sizes have slightly different semantics from the generated code, in that it
+// always returns size==1 for an illegal UTF-8 byte (instead of the length
+// of the maximum invalid subsequence). Most Transformers, like unicode/norm,
+// leave invalid UTF-8 untouched, in which case it has performance benefits to
+// do so (without changing the semantics). Bidi requires the semantics used here
+// for the bidirule implementation to be compatible with the Go semantics.
+//  They ultimately should perhaps be adopted by all trie implementations, for
+// convenience sake.
+// This unrolled code also boosts performance of the secure/bidirule package by
+// about 30%.
+// So, to remove this code:
+//   - add option to trie generator to define return type.
+//   - always return 1 byte size for ill-formed UTF-8 runes.
+
+// Lookup returns properties for the first rune in s and the width in bytes of
+// its encoding. The size will be 0 if s does not hold enough bytes to complete
+// the encoding.
+func Lookup(s []byte) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}
+
+// LookupString returns properties for the first rune in s and the width in
+// bytes of its encoding. The size will be 0 if s does not hold enough bytes to
+// complete the encoding.
+func LookupString(s string) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}

src/vendor/golang_org/x/text/unicode/bidi/trieval.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Code generated by running "go generate" in golang_org/x/text. DO NOT EDIT.
+
+package bidi
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

src/vendor/golang_org/x/text/unicode/doc.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// unicode holds packages with implementations of Unicode standards that are
+// mostly used as building blocks for other packages in golang_org/x/text,
+// layout engines, or are otherwise more low-level in nature.
+package unicode

src/vendor/golang_org/x/text/unicode/norm/composition.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/unicode/norm/example_iter_test.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm_test
+
+import (
+	"bytes"
+	"fmt"
+	"unicode/utf8"
+
+	"golang_org/x/text/unicode/norm"
+)
+
+// EqualSimple uses a norm.Iter to compare two non-normalized
+// strings for equivalence.
+func EqualSimple(a, b string) bool {
+	var ia, ib norm.Iter
+	ia.InitString(norm.NFKD, a)
+	ib.InitString(norm.NFKD, b)
+	for !ia.Done() && !ib.Done() {
+		if !bytes.Equal(ia.Next(), ib.Next()) {
+			return false
+		}
+	}
+	return ia.Done() && ib.Done()
+}
+
+// FindPrefix finds the longest common prefix of ASCII characters
+// of a and b.
+func FindPrefix(a, b string) int {
+	i := 0
+	for ; i < len(a) && i < len(b) && a[i] < utf8.RuneSelf && a[i] == b[i]; i++ {
+	}
+	return i
+}
+
+// EqualOpt is like EqualSimple, but optimizes the special
+// case for ASCII characters.
+func EqualOpt(a, b string) bool {
+	n := FindPrefix(a, b)
+	a, b = a[n:], b[n:]
+	var ia, ib norm.Iter
+	ia.InitString(norm.NFKD, a)
+	ib.InitString(norm.NFKD, b)
+	for !ia.Done() && !ib.Done() {
+		if !bytes.Equal(ia.Next(), ib.Next()) {
+			return false
+		}
+		if n := int64(FindPrefix(a[ia.Pos():], b[ib.Pos():])); n != 0 {
+			ia.Seek(n, 1)
+			ib.Seek(n, 1)
+		}
+	}
+	return ia.Done() && ib.Done()
+}
+
+var compareTests = []struct{ a, b string }{
+	{"aaa", "aaa"},
+	{"aaa", "aab"},
+	{"a\u0300a", "\u00E0a"},
+	{"a\u0300\u0320b", "a\u0320\u0300b"},
+	{"\u1E0A\u0323", "\x44\u0323\u0307"},
+	// A character that decomposes into multiple segments
+	// spans several iterations.
+	{"\u3304", "\u30A4\u30CB\u30F3\u30AF\u3099"},
+}
+
+func ExampleIter() {
+	for i, t := range compareTests {
+		r0 := EqualSimple(t.a, t.b)
+		r1 := EqualOpt(t.a, t.b)
+		fmt.Printf("%d: %v %v\n", i, r0, r1)
+	}
+	// Output:
+	// 0: true true
+	// 1: false false
+	// 2: true true
+	// 3: true true
+	// 4: true true
+	// 5: true true
+}

src/vendor/golang_org/x/text/unicode/norm/example_test.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm_test
+
+import (
+	"fmt"
+
+	"golang_org/x/text/unicode/norm"
+)
+
+func ExampleForm_NextBoundary() {
+	s := norm.NFD.String("Mêlée")
+
+	for i := 0; i < len(s); {
+		d := norm.NFC.NextBoundaryInString(s[i:], true)
+		fmt.Printf("%[1]s: %+[1]q\n", s[i:i+d])
+		i += d
+	}
+	// Output:
+	// M: "M"
+	// ê: "e\u0302"
+	// l: "l"
+	// é: "e\u0301"
+	// e: "e"
+}

src/vendor/golang_org/x/text/unicode/norm/forminfo.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
@@ -10,7 +12,7 @@ package norm
 // and its corresponding decomposing form share the same trie.  Each trie maps
 // a rune to a uint16. The values take two forms.  For v >= 0x8000:
 //   bits
-//   15:    1 (inverse of NFD_QD bit of qcInfo)
+//   15:    1 (inverse of NFD_QC bit of qcInfo)
 //   13..7: qcInfo (see below). isYesD is always true (no decompostion).
 //    6..0: ccc (compressed CCC value).
 // For v < 0x8000, the respective rune has a decomposition and v is an index
@@ -56,28 +58,31 @@ type formInfo struct {
 	nextMain                 iterFunc
 }
 
-var formTable []*formInfo
-
-func init() {
-	formTable = make([]*formInfo, 4)
-
-	for i := range formTable {
-		f := &formInfo{}
-		formTable[i] = f
-		f.form = Form(i)
-		if Form(i) == NFKD || Form(i) == NFKC {
-			f.compatibility = true
-			f.info = lookupInfoNFKC
-		} else {
-			f.info = lookupInfoNFC
-		}
-		f.nextMain = nextDecomposed
-		if Form(i) == NFC || Form(i) == NFKC {
-			f.nextMain = nextComposed
-			f.composing = true
-		}
-	}
-}
+var formTable = []*formInfo{{
+	form:          NFC,
+	composing:     true,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFD,
+	composing:     false,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextDecomposed,
+}, {
+	form:          NFKC,
+	composing:     true,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFKD,
+	composing:     false,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextDecomposed,
+}}
 
 // We do not distinguish between boundaries for NFC, NFD, etc. to avoid
 // unexpected behavior for the user.  For example, in NFD, there is a boundary

src/vendor/golang_org/x/text/unicode/norm/input.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/unicode/norm/iter.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/unicode/norm/normalize.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go triegen.go
-//go:generate go run maketables.go triegen.go -test
+// Note: the file data_test.go that is generated should not be checked in.
 
 // Package norm contains types and functions for normalizing Unicode strings.
-package norm // import "golang.org/x/text/unicode/norm"
+package norm // import "golang_org/x/text/unicode/norm"
 
 import (
 	"unicode/utf8"

src/vendor/golang_org/x/text/unicode/norm/readwriter.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/unicode/norm/transform.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2013 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/unicode/norm/trie.go

+// Code generated by running "go run gen.go -core" in golang.org/x/text. DO NOT EDIT.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

src/vendor/golang_org/x/text/width/kind_string.go

-// Code generated by "stringer -type=Kind"; DO NOT EDIT
-
-package width
-
-import "fmt"
-
-const _Kind_name = "NeutralEastAsianAmbiguousEastAsianWideEastAsianNarrowEastAsianFullwidthEastAsianHalfwidth"
-
-var _Kind_index = [...]uint8{0, 7, 25, 38, 53, 71, 89}
-
-func (i Kind) String() string {
-	if i < 0 || i >= Kind(len(_Kind_index)-1) {
-		return fmt.Sprintf("Kind(%d)", i)
-	}
-	return _Kind_name[_Kind_index[i]:_Kind_index[i+1]]
-}

src/vendor/golang_org/x/text/width/transform.go

-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package width
-
-import (
-	"unicode/utf8"
-
-	"golang_org/x/text/transform"
-)
-
-type foldTransform struct {
-	transform.NopResetter
-}
-
-func (foldTransform) Span(src []byte, atEOF bool) (n int, err error) {
-	for n < len(src) {
-		if src[n] < utf8.RuneSelf {
-			// ASCII fast path.
-			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
-			}
-			continue
-		}
-		v, size := trie.lookup(src[n:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				err = transform.ErrShortSrc
-			} else {
-				n = len(src)
-			}
-			break
-		}
-		if elem(v)&tagNeedsFold != 0 {
-			err = transform.ErrEndOfSpan
-			break
-		}
-		n += size
-	}
-	return n, err
-}
-
-func (foldTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
-	for nSrc < len(src) {
-		if src[nSrc] < utf8.RuneSelf {
-			// ASCII fast path.
-			start, end := nSrc, len(src)
-			if d := len(dst) - nDst; d < end-start {
-				end = nSrc + d
-			}
-			for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ {
-			}
-			n := copy(dst[nDst:], src[start:nSrc])
-			if nDst += n; nDst == len(dst) {
-				nSrc = start + n
-				if nSrc == len(src) {
-					return nDst, nSrc, nil
-				}
-				if src[nSrc] < utf8.RuneSelf {
-					return nDst, nSrc, transform.ErrShortDst
-				}
-			}
-			continue
-		}
-		v, size := trie.lookup(src[nSrc:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				return nDst, nSrc, transform.ErrShortSrc
-			}
-			size = 1 // gobble 1 byte
-		}
-		if elem(v)&tagNeedsFold == 0 {
-			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			nDst += size
-		} else {
-			data := inverseData[byte(v)]
-			if len(dst)-nDst < int(data[0]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			i := 1
-			for end := int(data[0]); i < end; i++ {
-				dst[nDst] = data[i]
-				nDst++
-			}
-			dst[nDst] = data[i] ^ src[nSrc+size-1]
-			nDst++
-		}
-		nSrc += size
-	}
-	return nDst, nSrc, nil
-}
-
-type narrowTransform struct {
-	transform.NopResetter
-}
-
-func (narrowTransform) Span(src []byte, atEOF bool) (n int, err error) {
-	for n < len(src) {
-		if src[n] < utf8.RuneSelf {
-			// ASCII fast path.
-			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
-			}
-			continue
-		}
-		v, size := trie.lookup(src[n:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				err = transform.ErrShortSrc
-			} else {
-				n = len(src)
-			}
-			break
-		}
-		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous {
-		} else {
-			err = transform.ErrEndOfSpan
-			break
-		}
-		n += size
-	}
-	return n, err
-}
-
-func (narrowTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
-	for nSrc < len(src) {
-		if src[nSrc] < utf8.RuneSelf {
-			// ASCII fast path.
-			start, end := nSrc, len(src)
-			if d := len(dst) - nDst; d < end-start {
-				end = nSrc + d
-			}
-			for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ {
-			}
-			n := copy(dst[nDst:], src[start:nSrc])
-			if nDst += n; nDst == len(dst) {
-				nSrc = start + n
-				if nSrc == len(src) {
-					return nDst, nSrc, nil
-				}
-				if src[nSrc] < utf8.RuneSelf {
-					return nDst, nSrc, transform.ErrShortDst
-				}
-			}
-			continue
-		}
-		v, size := trie.lookup(src[nSrc:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				return nDst, nSrc, transform.ErrShortSrc
-			}
-			size = 1 // gobble 1 byte
-		}
-		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous {
-			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			nDst += size
-		} else {
-			data := inverseData[byte(v)]
-			if len(dst)-nDst < int(data[0]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			i := 1
-			for end := int(data[0]); i < end; i++ {
-				dst[nDst] = data[i]
-				nDst++
-			}
-			dst[nDst] = data[i] ^ src[nSrc+size-1]
-			nDst++
-		}
-		nSrc += size
-	}
-	return nDst, nSrc, nil
-}
-
-type wideTransform struct {
-	transform.NopResetter
-}
-
-func (wideTransform) Span(src []byte, atEOF bool) (n int, err error) {
-	for n < len(src) {
-		// TODO: Consider ASCII fast path. Special-casing ASCII handling can
-		// reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably
-		// not enough to warrant the extra code and complexity.
-		v, size := trie.lookup(src[n:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				err = transform.ErrShortSrc
-			} else {
-				n = len(src)
-			}
-			break
-		}
-		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow {
-		} else {
-			err = transform.ErrEndOfSpan
-			break
-		}
-		n += size
-	}
-	return n, err
-}
-
-func (wideTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
-	for nSrc < len(src) {
-		// TODO: Consider ASCII fast path. Special-casing ASCII handling can
-		// reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably
-		// not enough to warrant the extra code and complexity.
-		v, size := trie.lookup(src[nSrc:])
-		if size == 0 { // incomplete UTF-8 encoding
-			if !atEOF {
-				return nDst, nSrc, transform.ErrShortSrc
-			}
-			size = 1 // gobble 1 byte
-		}
-		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow {
-			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			nDst += size
-		} else {
-			data := inverseData[byte(v)]
-			if len(dst)-nDst < int(data[0]) {
-				return nDst, nSrc, transform.ErrShortDst
-			}
-			i := 1
-			for end := int(data[0]); i < end; i++ {
-				dst[nDst] = data[i]
-				nDst++
-			}
-			dst[nDst] = data[i] ^ src[nSrc+size-1]
-			nDst++
-		}
-		nSrc += size
-	}
-	return nDst, nSrc, nil
-}

src/vendor/golang_org/x/text/width/trieval.go

-// This file was generated by go generate; DO NOT EDIT
-
-package width
-
-// elem is an entry of the width trie. The high byte is used to encode the type
-// of the rune. The low byte is used to store the index to a mapping entry in
-// the inverseData array.
-type elem uint16
-
-const (
-	tagNeutral elem = iota << typeShift
-	tagAmbiguous
-	tagWide
-	tagNarrow
-	tagFullwidth
-	tagHalfwidth
-)
-
-const (
-	numTypeBits = 3
-	typeShift   = 16 - numTypeBits
-
-	// tagNeedsFold is true for all fullwidth and halfwidth runes except for
-	// the Won sign U+20A9.
-	tagNeedsFold = 0x1000
-
-	// The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide
-	// variant.
-	wonSign rune = 0x20A9
-)

src/vendor/golang_org/x/text/width/width.go

-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:generate stringer -type=Kind
-//go:generate go run gen.go gen_common.go gen_trieval.go
-
-// Package width provides functionality for handling different widths in text.
-//
-// Wide characters behave like ideographs; they tend to allow line breaks after
-// each character and remain upright in vertical text layout. Narrow characters
-// are kept together in words or runs that are rotated sideways in vertical text
-// layout.
-//
-// For more information, see http://unicode.org/reports/tr11/.
-package width // import "golang_org/x/text/width"
-
-import (
-	"unicode/utf8"
-
-	"golang_org/x/text/transform"
-)
-
-// TODO
-// 1) Reduce table size by compressing blocks.
-// 2) API proposition for computing display length
-//    (approximation, fixed pitch only).
-// 3) Implement display length.
-
-// Kind indicates the type of width property as defined in http://unicode.org/reports/tr11/.
-type Kind int
-
-const (
-	// Neutral characters do not occur in legacy East Asian character sets.
-	Neutral Kind = iota
-
-	// EastAsianAmbiguous characters that can be sometimes wide and sometimes
-	// narrow and require additional information not contained in the character
-	// code to further resolve their width.
-	EastAsianAmbiguous
-
-	// EastAsianWide characters are wide in its usual form. They occur only in
-	// the context of East Asian typography. These runes may have explicit
-	// halfwidth counterparts.
-	EastAsianWide
-
-	// EastAsianNarrow characters are narrow in its usual form. They often have
-	// fullwidth counterparts.
-	EastAsianNarrow
-
-	// Note: there exist Narrow runes that do not have fullwidth or wide
-	// counterparts, despite what the definition says (e.g. U+27E6).
-
-	// EastAsianFullwidth characters have a compatibility decompositions of type
-	// wide that map to a narrow counterpart.
-	EastAsianFullwidth
-
-	// EastAsianHalfwidth characters have a compatibility decomposition of type
-	// narrow that map to a wide or ambiguous counterpart, plus U+20A9 ₩ WON
-	// SIGN.
-	EastAsianHalfwidth
-
-	// Note: there exist runes that have a halfwidth counterparts but that are
-	// classified as Ambiguous, rather than wide (e.g. U+2190).
-)
-
-// TODO: the generated tries need to return size 1 for invalid runes for the
-// width to be computed correctly (each byte should render width 1)
-
-var trie = newWidthTrie(0)
-
-// Lookup reports the Properties of the first rune in b and the number of bytes
-// of its UTF-8 encoding.
-func Lookup(b []byte) (p Properties, size int) {
-	v, sz := trie.lookup(b)
-	return Properties{elem(v), b[sz-1]}, sz
-}
-
-// LookupString reports the Properties of the first rune in s and the number of
-// bytes of its UTF-8 encoding.
-func LookupString(s string) (p Properties, size int) {
-	v, sz := trie.lookupString(s)
-	return Properties{elem(v), s[sz-1]}, sz
-}
-
-// LookupRune reports the Properties of rune r.
-func LookupRune(r rune) Properties {
-	var buf [4]byte
-	n := utf8.EncodeRune(buf[:], r)
-	v, _ := trie.lookup(buf[:n])
-	last := byte(r)
-	if r >= utf8.RuneSelf {
-		last = 0x80 + byte(r&0x3f)
-	}
-	return Properties{elem(v), last}
-}
-
-// Properties provides access to width properties of a rune.
-type Properties struct {
-	elem elem
-	last byte
-}
-
-func (e elem) kind() Kind {
-	return Kind(e >> typeShift)
-}
-
-// Kind returns the Kind of a rune as defined in Unicode TR #11.
-// See http://unicode.org/reports/tr11/ for more details.
-func (p Properties) Kind() Kind {
-	return p.elem.kind()
-}
-
-// Folded returns the folded variant of a rune or 0 if the rune is canonical.
-func (p Properties) Folded() rune {
-	if p.elem&tagNeedsFold != 0 {
-		buf := inverseData[byte(p.elem)]
-		buf[buf[0]] ^= p.last
-		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
-		return r
-	}
-	return 0
-}
-
-// Narrow returns the narrow variant of a rune or 0 if the rune is already
-// narrow or doesn't have a narrow variant.
-func (p Properties) Narrow() rune {
-	if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous) {
-		buf := inverseData[byte(p.elem)]
-		buf[buf[0]] ^= p.last
-		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
-		return r
-	}
-	return 0
-}
-
-// Wide returns the wide variant of a rune or 0 if the rune is already
-// wide or doesn't have a wide variant.
-func (p Properties) Wide() rune {
-	if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianHalfwidth || k == EastAsianNarrow) {
-		buf := inverseData[byte(p.elem)]
-		buf[buf[0]] ^= p.last
-		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
-		return r
-	}
-	return 0
-}
-
-// TODO for Properties:
-// - Add Fullwidth/Halfwidth or Inverted methods for computing variants
-// mapping.
-// - Add width information (including information on non-spacing runes).
-
-// Transformer implements the transform.Transformer interface.
-type Transformer struct {
-	t transform.SpanningTransformer
-}
-
-// Reset implements the transform.Transformer interface.
-func (t Transformer) Reset() { t.t.Reset() }
-
-// Transform implements the transform.Transformer interface.
-func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
-	return t.t.Transform(dst, src, atEOF)
-}
-
-// Span implements the transform.SpanningTransformer interface.
-func (t Transformer) Span(src []byte, atEOF bool) (n int, err error) {
-	return t.t.Span(src, atEOF)
-}
-
-// Bytes returns a new byte slice with the result of applying t to b.
-func (t Transformer) Bytes(b []byte) []byte {
-	b, _, _ = transform.Bytes(t, b)
-	return b
-}
-
-// String returns a string with the result of applying t to s.
-func (t Transformer) String(s string) string {
-	s, _, _ = transform.String(t, s)
-	return s
-}
-
-var (
-	// Fold is a transform that maps all runes to their canonical width.
-	//
-	// Note that the NFKC and NFKD transforms in golang.org/x/text/unicode/norm
-	// provide a more generic folding mechanism.
-	Fold Transformer = Transformer{foldTransform{}}
-
-	// Widen is a transform that maps runes to their wide variant, if
-	// available.
-	Widen Transformer = Transformer{wideTransform{}}
-
-	// Narrow is a transform that maps runes to their narrow variant, if
-	// available.
-	Narrow Transformer = Transformer{narrowTransform{}}
-)
-
-// TODO: Consider the following options:
-// - Treat Ambiguous runes that have a halfwidth counterpart as wide, or some
-//   generalized variant of this.
-// - Consider a wide Won character to be the default width (or some generalized
-//   variant of this).
-// - Filter the set of characters that gets converted (the preferred approach is
-//   to allow applying filters to transforms).