vendor: add golang.org/x/text/unicode/norm + x/test/width for IDNA support

Add golang.org/x/text/unicode/norm from x/text git rev a7c02369.

Needed by net/http for IDNA normalization.

Updates #13835

Change-Id: I8b024e179d573f2b093c209a4b9e4f71f7d4a1f2
Reviewed-on: https://go-review.googlesource.com/29859
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

src/vendor/golang_org/x/text/unicode/norm/forminfo.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+// This file contains Form-specific logic and wrappers for data in tables.go.
+
+// Rune info is stored in a separate trie per composing form. A composing form
+// and its corresponding decomposing form share the same trie.  Each trie maps
+// a rune to a uint16. The values take two forms.  For v >= 0x8000:
+//   bits
+//   15:    1 (inverse of NFD_QD bit of qcInfo)
+//   13..7: qcInfo (see below). isYesD is always true (no decompostion).
+//    6..0: ccc (compressed CCC value).
+// For v < 0x8000, the respective rune has a decomposition and v is an index
+// into a byte array of UTF-8 decomposition sequences and additional info and
+// has the form:
+//    <header> <decomp_byte>* [<tccc> [<lccc>]]
+// The header contains the number of bytes in the decomposition (excluding this
+// length byte). The two most significant bits of this length byte correspond
+// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// The byte sequence is followed by a trailing and leading CCC if the values
+// for these are not zero.  The value of v determines which ccc are appended
+// to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
+// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
+// there is an additional leading ccc. The value of tccc itself is the
+// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
+// are the number of trailing non-starters.
+
+const (
+	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
+	headerLenMask   = 0x3F // extract the length value from the header byte
+	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+)
+
+// Properties provides access to normalization properties of a rune.
+type Properties struct {
+	pos   uint8  // start position in reorderBuffer; used in composition.go
+	size  uint8  // length of UTF-8 encoding of this rune
+	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
+	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
+	nLead uint8  // number of leading non-starters.
+	flags qcInfo // quick check flags
+	index uint16
+}
+
+// functions dispatchable per form
+type lookupFunc func(b input, i int) Properties
+
+// formInfo holds Form-specific functions and tables.
+type formInfo struct {
+	form                     Form
+	composing, compatibility bool // form type
+	info                     lookupFunc
+	nextMain                 iterFunc
+}
+
+var formTable []*formInfo
+
+func init() {
+	formTable = make([]*formInfo, 4)
+
+	for i := range formTable {
+		f := &formInfo{}
+		formTable[i] = f
+		f.form = Form(i)
+		if Form(i) == NFKD || Form(i) == NFKC {
+			f.compatibility = true
+			f.info = lookupInfoNFKC
+		} else {
+			f.info = lookupInfoNFC
+		}
+		f.nextMain = nextDecomposed
+		if Form(i) == NFC || Form(i) == NFKC {
+			f.nextMain = nextComposed
+			f.composing = true
+		}
+	}
+}
+
+// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
+// unexpected behavior for the user.  For example, in NFD, there is a boundary
+// after 'a'.  However, 'a' might combine with modifiers, so from the application's
+// perspective it is not a good boundary. We will therefore always use the
+// boundaries for the combining variants.
+
+// BoundaryBefore returns true if this rune starts a new segment and
+// cannot combine with any rune on the left.
+func (p Properties) BoundaryBefore() bool {
+	if p.ccc == 0 && !p.combinesBackward() {
+		return true
+	}
+	// We assume that the CCC of the first character in a decomposition
+	// is always non-zero if different from info.ccc and that we can return
+	// false at this point. This is verified by maketables.
+	return false
+}
+
+// BoundaryAfter returns true if runes cannot combine with or otherwise
+// interact with this or previous runes.
+func (p Properties) BoundaryAfter() bool {
+	// TODO: loosen these conditions.
+	return p.isInert()
+}
+
+// We pack quick check data in 4 bits:
+//   5:    Combines forward  (0 == false, 1 == true)
+//   4..3: NFC_QC Yes(00), No (10), or Maybe (11)
+//   2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
+//   1..0: Number of trailing non-starters.
+//
+// When all 4 bits are zero, the character is inert, meaning it is never
+// influenced by normalization.
+type qcInfo uint8
+
+func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
+func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
+
+func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
+func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
+func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
+
+func (p Properties) isInert() bool {
+	return p.flags&qcInfoMask == 0 && p.ccc == 0
+}
+
+func (p Properties) multiSegment() bool {
+	return p.index >= firstMulti && p.index < endMulti
+}
+
+func (p Properties) nLeadingNonStarters() uint8 {
+	return p.nLead
+}
+
+func (p Properties) nTrailingNonStarters() uint8 {
+	return uint8(p.flags & 0x03)
+}
+
+// Decomposition returns the decomposition for the underlying rune
+// or nil if there is none.
+func (p Properties) Decomposition() []byte {
+	// TODO: create the decomposition for Hangul?
+	if p.index == 0 {
+		return nil
+	}
+	i := p.index
+	n := decomps[i] & headerLenMask
+	i++
+	return decomps[i : i+uint16(n)]
+}
+
+// Size returns the length of UTF-8 encoding of the rune.
+func (p Properties) Size() int {
+	return int(p.size)
+}
+
+// CCC returns the canonical combining class of the underlying rune.
+func (p Properties) CCC() uint8 {
+	if p.index >= firstCCCZeroExcept {
+		return 0
+	}
+	return ccc[p.ccc]
+}
+
+// LeadCCC returns the CCC of the first rune in the decomposition.
+// If there is no decomposition, LeadCCC equals CCC.
+func (p Properties) LeadCCC() uint8 {
+	return ccc[p.ccc]
+}
+
+// TrailCCC returns the CCC of the last rune in the decomposition.
+// If there is no decomposition, TrailCCC equals CCC.
+func (p Properties) TrailCCC() uint8 {
+	return ccc[p.tccc]
+}
+
+// Recomposition
+// We use 32-bit keys instead of 64-bit for the two codepoint keys.
+// This clips off the bits of three entries, but we know this will not
+// result in a collision. In the unlikely event that changes to
+// UnicodeData.txt introduce collisions, the compiler will catch it.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+// combine returns the combined rune or 0 if it doesn't exist.
+func combine(a, b rune) rune {
+	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	return recompMap[key]
+}
+
+func lookupInfoNFC(b input, i int) Properties {
+	v, sz := b.charinfoNFC(i)
+	return compInfo(v, sz)
+}
+
+func lookupInfoNFKC(b input, i int) Properties {
+	v, sz := b.charinfoNFKC(i)
+	return compInfo(v, sz)
+}
+
+// Properties returns properties for the first rune in s.
+func (f Form) Properties(s []byte) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookup(s))
+	}
+	return compInfo(nfkcData.lookup(s))
+}
+
+// PropertiesString returns properties for the first rune in s.
+func (f Form) PropertiesString(s string) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookupString(s))
+	}
+	return compInfo(nfkcData.lookupString(s))
+}
+
+// compInfo converts the information contained in v and sz
+// to a Properties.  See the comment at the top of the file
+// for more information on the format.
+func compInfo(v uint16, sz int) Properties {
+	if v == 0 {
+		return Properties{size: uint8(sz)}
+	} else if v >= 0x8000 {
+		p := Properties{
+			size:  uint8(sz),
+			ccc:   uint8(v),
+			tccc:  uint8(v),
+			flags: qcInfo(v >> 8),
+		}
+		if p.ccc > 0 || p.combinesBackward() {
+			p.nLead = uint8(p.flags & 0x3)
+		}
+		return p
+	}
+	// has decomposition
+	h := decomps[v]
+	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
+	p := Properties{size: uint8(sz), flags: f, index: v}
+	if v >= firstCCC {
+		v += uint16(h&headerLenMask) + 1
+		c := decomps[v]
+		p.tccc = c >> 2
+		p.flags |= qcInfo(c & 0x3)
+		if v >= firstLeadingCCC {
+			p.nLead = c & 0x3
+			if v >= firstStarterWithNLead {
+				// We were tricked. Remove the decomposition.
+				p.flags &= 0x03
+				p.index = 0
+				return p
+			}
+			p.ccc = decomps[v+1]
+		}
+	}
+	return p
+}

src/vendor/golang_org/x/text/unicode/norm/input.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+type input struct {
+	str   string
+	bytes []byte
+}
+
+func inputBytes(str []byte) input {
+	return input{bytes: str}
+}
+
+func inputString(str string) input {
+	return input{str: str}
+}
+
+func (in *input) setBytes(str []byte) {
+	in.str = ""
+	in.bytes = str
+}
+
+func (in *input) setString(str string) {
+	in.str = str
+	in.bytes = nil
+}
+
+func (in *input) _byte(p int) byte {
+	if in.bytes == nil {
+		return in.str[p]
+	}
+	return in.bytes[p]
+}
+
+func (in *input) skipASCII(p, max int) int {
+	if in.bytes == nil {
+		for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
+		}
+	} else {
+		for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) skipContinuationBytes(p int) int {
+	if in.bytes == nil {
+		for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
+		}
+	} else {
+		for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) appendSlice(buf []byte, b, e int) []byte {
+	if in.bytes != nil {
+		return append(buf, in.bytes[b:e]...)
+	}
+	for i := b; i < e; i++ {
+		buf = append(buf, in.str[i])
+	}
+	return buf
+}
+
+func (in *input) copySlice(buf []byte, b, e int) int {
+	if in.bytes == nil {
+		return copy(buf, in.str[b:e])
+	}
+	return copy(buf, in.bytes[b:e])
+}
+
+func (in *input) charinfoNFC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfcData.lookupString(in.str[p:])
+	}
+	return nfcData.lookup(in.bytes[p:])
+}
+
+func (in *input) charinfoNFKC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfkcData.lookupString(in.str[p:])
+	}
+	return nfkcData.lookup(in.bytes[p:])
+}
+
+func (in *input) hangul(p int) (r rune) {
+	if in.bytes == nil {
+		if !isHangulString(in.str[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRuneInString(in.str[p:])
+	} else {
+		if !isHangul(in.bytes[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRune(in.bytes[p:])
+	}
+	return r
+}

src/vendor/golang_org/x/text/unicode/norm/readwriter.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "io"
+
+type normWriter struct {
+	rb  reorderBuffer
+	w   io.Writer
+	buf []byte
+}
+
+// Write implements the standard write interface.  If the last characters are
+// not at a normalization boundary, the bytes will be buffered for the next
+// write. The remaining bytes will be written on close.
+func (w *normWriter) Write(data []byte) (n int, err error) {
+	// Process data in pieces to keep w.buf size bounded.
+	const chunk = 4000
+
+	for len(data) > 0 {
+		// Normalize into w.buf.
+		m := len(data)
+		if m > chunk {
+			m = chunk
+		}
+		w.rb.src = inputBytes(data[:m])
+		w.rb.nsrc = m
+		w.buf = doAppend(&w.rb, w.buf, 0)
+		data = data[m:]
+		n += m
+
+		// Write out complete prefix, save remainder.
+		// Note that lastBoundary looks back at most 31 runes.
+		i := lastBoundary(&w.rb.f, w.buf)
+		if i == -1 {
+			i = 0
+		}
+		if i > 0 {
+			if _, err = w.w.Write(w.buf[:i]); err != nil {
+				break
+			}
+			bn := copy(w.buf, w.buf[i:])
+			w.buf = w.buf[:bn]
+		}
+	}
+	return n, err
+}
+
+// Close forces data that remains in the buffer to be written.
+func (w *normWriter) Close() error {
+	if len(w.buf) > 0 {
+		_, err := w.w.Write(w.buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Writer returns a new writer that implements Write(b)
+// by writing f(b) to w.  The returned writer may use an
+// an internal buffer to maintain state across Write calls.
+// Calling its Close method writes any buffered data to w.
+func (f Form) Writer(w io.Writer) io.WriteCloser {
+	wr := &normWriter{rb: reorderBuffer{}, w: w}
+	wr.rb.init(f, nil)
+	return wr
+}
+
+type normReader struct {
+	rb           reorderBuffer
+	r            io.Reader
+	inbuf        []byte
+	outbuf       []byte
+	bufStart     int
+	lastBoundary int
+	err          error
+}
+
+// Read implements the standard read interface.
+func (r *normReader) Read(p []byte) (int, error) {
+	for {
+		if r.lastBoundary-r.bufStart > 0 {
+			n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
+			r.bufStart += n
+			if r.lastBoundary-r.bufStart > 0 {
+				return n, nil
+			}
+			return n, r.err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
+		r.outbuf = r.outbuf[0:outn]
+		r.bufStart = 0
+
+		n, err := r.r.Read(r.inbuf)
+		r.rb.src = inputBytes(r.inbuf[0:n])
+		r.rb.nsrc, r.err = n, err
+		if n > 0 {
+			r.outbuf = doAppend(&r.rb, r.outbuf, 0)
+		}
+		if err == io.EOF {
+			r.lastBoundary = len(r.outbuf)
+		} else {
+			r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
+			if r.lastBoundary == -1 {
+				r.lastBoundary = 0
+			}
+		}
+	}
+	panic("should not reach here")
+}
+
+// Reader returns a new reader that implements Read
+// by reading data from r and returning f(data).
+func (f Form) Reader(r io.Reader) io.Reader {
+	const chunk = 4000
+	buf := make([]byte, chunk)
+	rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
+	rr.rb.init(f, buf)
+	return rr
+}

src/vendor/golang_org/x/text/unicode/norm/transform.go

+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"unicode/utf8"
+
+	"golang_org/x/text/transform"
+)
+
+// Reset implements the Reset method of the transform.Transformer interface.
+func (Form) Reset() {}
+
+// Transform implements the Transform method of the transform.Transformer
+// interface. It may need to write segments of up to MaxSegmentSize at once.
+// Users should either catch ErrShortDst and allow dst to grow or have dst be at
+// least of size MaxTransformChunkSize to be guaranteed of progress.
+func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := 0
+	// Cap the maximum number of src bytes to check.
+	b := src
+	eof := atEOF
+	if ns := len(dst); ns < len(b) {
+		err = transform.ErrShortDst
+		eof = false
+		b = b[:ns]
+	}
+	i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
+	n += copy(dst[n:], b[n:i])
+	if !ok {
+		nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
+		return nDst + n, nSrc + n, err
+	}
+	if n < len(src) && !atEOF {
+		err = transform.ErrShortSrc
+	}
+	return n, n, err
+}
+
+func flushTransform(rb *reorderBuffer) bool {
+	// Write out (must fully fit in dst, or else it is a ErrShortDst).
+	if len(rb.out) < rb.nrune*utf8.UTFMax {
+		return false
+	}
+	rb.out = rb.out[rb.flushCopy(rb.out):]
+	return true
+}
+
+var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
+
+// transform implements the transform.Transformer interface. It is only called
+// when quickSpan does not pass for a given string.
+func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// TODO: get rid of reorderBuffer. See CL 23460044.
+	rb := reorderBuffer{}
+	rb.init(f, src)
+	for {
+		// Load segment into reorder buffer.
+		rb.setFlusher(dst[nDst:], flushTransform)
+		end := decomposeSegment(&rb, nSrc, atEOF)
+		if end < 0 {
+			return nDst, nSrc, errs[-end]
+		}
+		nDst = len(dst) - len(rb.out)
+		nSrc = end
+
+		// Next quickSpan.
+		end = rb.nsrc
+		eof := atEOF
+		if n := nSrc + len(dst) - nDst; n < end {
+			err = transform.ErrShortDst
+			end = n
+			eof = false
+		}
+		end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
+		n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
+		nSrc += n
+		nDst += n
+		if ok {
+			if n < rb.nsrc && !atEOF {
+				err = transform.ErrShortSrc
+			}
+			return nDst, nSrc, err
+		}
+	}
+}

src/vendor/golang_org/x/text/unicode/norm/trie.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var nfcSparse = sparseBlocks{
+	values: nfcSparseValues[:],
+	offset: nfcSparseOffset[:],
+}
+
+var nfkcSparse = sparseBlocks{
+	values: nfkcSparseValues[:],
+	offset: nfkcSparseOffset[:],
+}
+
+var (
+	nfcData  = newNfcTrie(0)
+	nfkcData = newNfkcTrie(0)
+)
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

src/vendor/golang_org/x/text/unicode/norm/triegen.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Trie table generator.
+// Used by make*tables tools to generate a go file with trie data structures
+// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table with 16-bit values.
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+const maxSparseEntries = 16
+
+type normCompacter struct {
+	sparseBlocks [][]uint64
+	sparseOffset []uint16
+	sparseCount  int
+	name         string
+}
+
+func mostFrequentStride(a []uint64) int {
+	counts := make(map[int]int)
+	var v int
+	for _, x := range a {
+		if stride := int(x) - v; v != 0 && stride >= 0 {
+			counts[stride]++
+		}
+		v = int(x)
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func countSparseEntries(a []uint64) int {
+	stride := mostFrequentStride(a)
+	var v, count int
+	for _, tv := range a {
+		if int(tv)-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = int(tv)
+	}
+	return count
+}
+
+func (c *normCompacter) Size(v []uint64) (sz int, ok bool) {
+	if n := countSparseEntries(v); n <= maxSparseEntries {
+		return (n+1)*4 + 2, true
+	}
+	return 0, false
+}
+
+func (c *normCompacter) Store(v []uint64) uint32 {
+	h := uint32(len(c.sparseOffset))
+	c.sparseBlocks = append(c.sparseBlocks, v)
+	c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount))
+	c.sparseCount += countSparseEntries(v) + 1
+	return h
+}
+
+func (c *normCompacter) Handler() string {
+	return c.name + "Sparse.lookup"
+}
+
+func (c *normCompacter) Print(w io.Writer) (retErr error) {
+	p := func(f string, x ...interface{}) {
+		if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil {
+			retErr = err
+		}
+	}
+
+	ls := len(c.sparseBlocks)
+	p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2)
+	p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset)
+
+	ns := c.sparseCount
+	p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4)
+	p("var %sSparseValues = [%d]valueRange {", c.name, ns)
+	for i, b := range c.sparseBlocks {
+		p("\n// Block %#x, offset %#x", i, c.sparseOffset[i])
+		var v int
+		stride := mostFrequentStride(b)
+		n := countSparseEntries(b)
+		p("\n{value:%#04x,lo:%#02x},", stride, uint8(n))
+		for i, nv := range b {
+			if int(nv)-v != stride {
+				if v != 0 {
+					p(",hi:%#02x},", 0x80+i-1)
+				}
+				if nv != 0 {
+					p("\n{value:%#04x,lo:%#02x", nv, 0x80+i)
+				}
+			}
+			v = int(nv)
+		}
+		if v != 0 {
+			p(",hi:%#02x},", 0x80+len(b)-1)
+		}
+	}
+	p("\n}\n\n")
+	return
+}

src/vendor/golang_org/x/text/width/kind_string.go

+// Code generated by "stringer -type=Kind"; DO NOT EDIT
+
+package width
+
+import "fmt"
+
+const _Kind_name = "NeutralEastAsianAmbiguousEastAsianWideEastAsianNarrowEastAsianFullwidthEastAsianHalfwidth"
+
+var _Kind_index = [...]uint8{0, 7, 25, 38, 53, 71, 89}
+
+func (i Kind) String() string {
+	if i < 0 || i >= Kind(len(_Kind_index)-1) {
+		return fmt.Sprintf("Kind(%d)", i)
+	}
+	return _Kind_name[_Kind_index[i]:_Kind_index[i+1]]
+}

src/vendor/golang_org/x/text/width/transform.go

+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package width
+
+import (
+	"unicode/utf8"
+
+	"golang_org/x/text/transform"
+)
+
+type foldTransform struct {
+	transform.NopResetter
+}
+
+func (foldTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		if src[n] < utf8.RuneSelf {
+			// ASCII fast path.
+			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
+			}
+			continue
+		}
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if elem(v)&tagNeedsFold != 0 {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
+func (foldTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for nSrc < len(src) {
+		if src[nSrc] < utf8.RuneSelf {
+			// ASCII fast path.
+			start, end := nSrc, len(src)
+			if d := len(dst) - nDst; d < end-start {
+				end = nSrc + d
+			}
+			for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ {
+			}
+			n := copy(dst[nDst:], src[start:nSrc])
+			if nDst += n; nDst == len(dst) {
+				nSrc = start + n
+				if nSrc == len(src) {
+					return nDst, nSrc, nil
+				}
+				if src[nSrc] < utf8.RuneSelf {
+					return nDst, nSrc, transform.ErrShortDst
+				}
+			}
+			continue
+		}
+		v, size := trie.lookup(src[nSrc:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				return nDst, nSrc, transform.ErrShortSrc
+			}
+			size = 1 // gobble 1 byte
+		}
+		if elem(v)&tagNeedsFold == 0 {
+			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			nDst += size
+		} else {
+			data := inverseData[byte(v)]
+			if len(dst)-nDst < int(data[0]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			i := 1
+			for end := int(data[0]); i < end; i++ {
+				dst[nDst] = data[i]
+				nDst++
+			}
+			dst[nDst] = data[i] ^ src[nSrc+size-1]
+			nDst++
+		}
+		nSrc += size
+	}
+	return nDst, nSrc, nil
+}
+
+type narrowTransform struct {
+	transform.NopResetter
+}
+
+func (narrowTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		if src[n] < utf8.RuneSelf {
+			// ASCII fast path.
+			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
+			}
+			continue
+		}
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous {
+		} else {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
+func (narrowTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for nSrc < len(src) {
+		if src[nSrc] < utf8.RuneSelf {
+			// ASCII fast path.
+			start, end := nSrc, len(src)
+			if d := len(dst) - nDst; d < end-start {
+				end = nSrc + d
+			}
+			for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ {
+			}
+			n := copy(dst[nDst:], src[start:nSrc])
+			if nDst += n; nDst == len(dst) {
+				nSrc = start + n
+				if nSrc == len(src) {
+					return nDst, nSrc, nil
+				}
+				if src[nSrc] < utf8.RuneSelf {
+					return nDst, nSrc, transform.ErrShortDst
+				}
+			}
+			continue
+		}
+		v, size := trie.lookup(src[nSrc:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				return nDst, nSrc, transform.ErrShortSrc
+			}
+			size = 1 // gobble 1 byte
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous {
+			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			nDst += size
+		} else {
+			data := inverseData[byte(v)]
+			if len(dst)-nDst < int(data[0]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			i := 1
+			for end := int(data[0]); i < end; i++ {
+				dst[nDst] = data[i]
+				nDst++
+			}
+			dst[nDst] = data[i] ^ src[nSrc+size-1]
+			nDst++
+		}
+		nSrc += size
+	}
+	return nDst, nSrc, nil
+}
+
+type wideTransform struct {
+	transform.NopResetter
+}
+
+func (wideTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		// TODO: Consider ASCII fast path. Special-casing ASCII handling can
+		// reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably
+		// not enough to warrant the extra code and complexity.
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow {
+		} else {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
+func (wideTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for nSrc < len(src) {
+		// TODO: Consider ASCII fast path. Special-casing ASCII handling can
+		// reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably
+		// not enough to warrant the extra code and complexity.
+		v, size := trie.lookup(src[nSrc:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				return nDst, nSrc, transform.ErrShortSrc
+			}
+			size = 1 // gobble 1 byte
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow {
+			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			nDst += size
+		} else {
+			data := inverseData[byte(v)]
+			if len(dst)-nDst < int(data[0]) {
+				return nDst, nSrc, transform.ErrShortDst
+			}
+			i := 1
+			for end := int(data[0]); i < end; i++ {
+				dst[nDst] = data[i]
+				nDst++
+			}
+			dst[nDst] = data[i] ^ src[nSrc+size-1]
+			nDst++
+		}
+		nSrc += size
+	}
+	return nDst, nSrc, nil
+}

src/vendor/golang_org/x/text/width/trieval.go

+// This file was generated by go generate; DO NOT EDIT
+
+package width
+
+// elem is an entry of the width trie. The high byte is used to encode the type
+// of the rune. The low byte is used to store the index to a mapping entry in
+// the inverseData array.
+type elem uint16
+
+const (
+	tagNeutral elem = iota << typeShift
+	tagAmbiguous
+	tagWide
+	tagNarrow
+	tagFullwidth
+	tagHalfwidth
+)
+
+const (
+	numTypeBits = 3
+	typeShift   = 16 - numTypeBits
+
+	// tagNeedsFold is true for all fullwidth and halfwidth runes except for
+	// the Won sign U+20A9.
+	tagNeedsFold = 0x1000
+
+	// The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide
+	// variant.
+	wonSign rune = 0x20A9
+)

src/vendor/golang_org/x/text/width/width.go

+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate stringer -type=Kind
+//go:generate go run gen.go gen_common.go gen_trieval.go
+
+// Package width provides functionality for handling different widths in text.
+//
+// Wide characters behave like ideographs; they tend to allow line breaks after
+// each character and remain upright in vertical text layout. Narrow characters
+// are kept together in words or runs that are rotated sideways in vertical text
+// layout.
+//
+// For more information, see http://unicode.org/reports/tr11/.
+package width // import "golang_org/x/text/width"
+
+import (
+	"unicode/utf8"
+
+	"golang_org/x/text/transform"
+)
+
+// TODO
+// 1) Reduce table size by compressing blocks.
+// 2) API proposition for computing display length
+//    (approximation, fixed pitch only).
+// 3) Implement display length.
+
+// Kind indicates the type of width property as defined in http://unicode.org/reports/tr11/.
+type Kind int
+
+const (
+	// Neutral characters do not occur in legacy East Asian character sets.
+	Neutral Kind = iota
+
+	// EastAsianAmbiguous characters that can be sometimes wide and sometimes
+	// narrow and require additional information not contained in the character
+	// code to further resolve their width.
+	EastAsianAmbiguous
+
+	// EastAsianWide characters are wide in its usual form. They occur only in
+	// the context of East Asian typography. These runes may have explicit
+	// halfwidth counterparts.
+	EastAsianWide
+
+	// EastAsianNarrow characters are narrow in its usual form. They often have
+	// fullwidth counterparts.
+	EastAsianNarrow
+
+	// Note: there exist Narrow runes that do not have fullwidth or wide
+	// counterparts, despite what the definition says (e.g. U+27E6).
+
+	// EastAsianFullwidth characters have a compatibility decompositions of type
+	// wide that map to a narrow counterpart.
+	EastAsianFullwidth
+
+	// EastAsianHalfwidth characters have a compatibility decomposition of type
+	// narrow that map to a wide or ambiguous counterpart, plus U+20A9 ₩ WON
+	// SIGN.
+	EastAsianHalfwidth
+
+	// Note: there exist runes that have a halfwidth counterparts but that are
+	// classified as Ambiguous, rather than wide (e.g. U+2190).
+)
+
+// TODO: the generated tries need to return size 1 for invalid runes for the
+// width to be computed correctly (each byte should render width 1)
+
+var trie = newWidthTrie(0)
+
+// Lookup reports the Properties of the first rune in b and the number of bytes
+// of its UTF-8 encoding.
+func Lookup(b []byte) (p Properties, size int) {
+	v, sz := trie.lookup(b)
+	return Properties{elem(v), b[sz-1]}, sz
+}
+
+// LookupString reports the Properties of the first rune in s and the number of
+// bytes of its UTF-8 encoding.
+func LookupString(s string) (p Properties, size int) {
+	v, sz := trie.lookupString(s)
+	return Properties{elem(v), s[sz-1]}, sz
+}
+
+// LookupRune reports the Properties of rune r.
+func LookupRune(r rune) Properties {
+	var buf [4]byte
+	n := utf8.EncodeRune(buf[:], r)
+	v, _ := trie.lookup(buf[:n])
+	last := byte(r)
+	if r >= utf8.RuneSelf {
+		last = 0x80 + byte(r&0x3f)
+	}
+	return Properties{elem(v), last}
+}
+
+// Properties provides access to width properties of a rune.
+type Properties struct {
+	elem elem
+	last byte
+}
+
+func (e elem) kind() Kind {
+	return Kind(e >> typeShift)
+}
+
+// Kind returns the Kind of a rune as defined in Unicode TR #11.
+// See http://unicode.org/reports/tr11/ for more details.
+func (p Properties) Kind() Kind {
+	return p.elem.kind()
+}
+
+// Folded returns the folded variant of a rune or 0 if the rune is canonical.
+func (p Properties) Folded() rune {
+	if p.elem&tagNeedsFold != 0 {
+		buf := inverseData[byte(p.elem)]
+		buf[buf[0]] ^= p.last
+		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
+		return r
+	}
+	return 0
+}
+
+// Narrow returns the narrow variant of a rune or 0 if the rune is already
+// narrow or doesn't have a narrow variant.
+func (p Properties) Narrow() rune {
+	if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous) {
+		buf := inverseData[byte(p.elem)]
+		buf[buf[0]] ^= p.last
+		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
+		return r
+	}
+	return 0
+}
+
+// Wide returns the wide variant of a rune or 0 if the rune is already
+// wide or doesn't have a wide variant.
+func (p Properties) Wide() rune {
+	if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianHalfwidth || k == EastAsianNarrow) {
+		buf := inverseData[byte(p.elem)]
+		buf[buf[0]] ^= p.last
+		r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]])
+		return r
+	}
+	return 0
+}
+
+// TODO for Properties:
+// - Add Fullwidth/Halfwidth or Inverted methods for computing variants
+// mapping.
+// - Add width information (including information on non-spacing runes).
+
+// Transformer implements the transform.Transformer interface.
+type Transformer struct {
+	t transform.SpanningTransformer
+}
+
+// Reset implements the transform.Transformer interface.
+func (t Transformer) Reset() { t.t.Reset() }
+
+// Transform implements the transform.Transformer interface.
+func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return t.t.Transform(dst, src, atEOF)
+}
+
+// Span implements the transform.SpanningTransformer interface.
+func (t Transformer) Span(src []byte, atEOF bool) (n int, err error) {
+	return t.t.Span(src, atEOF)
+}
+
+// Bytes returns a new byte slice with the result of applying t to b.
+func (t Transformer) Bytes(b []byte) []byte {
+	b, _, _ = transform.Bytes(t, b)
+	return b
+}
+
+// String returns a string with the result of applying t to s.
+func (t Transformer) String(s string) string {
+	s, _, _ = transform.String(t, s)
+	return s
+}
+
+var (
+	// Fold is a transform that maps all runes to their canonical width.
+	//
+	// Note that the NFKC and NFKD transforms in golang.org/x/text/unicode/norm
+	// provide a more generic folding mechanism.
+	Fold Transformer = Transformer{foldTransform{}}
+
+	// Widen is a transform that maps runes to their wide variant, if
+	// available.
+	Widen Transformer = Transformer{wideTransform{}}
+
+	// Narrow is a transform that maps runes to their narrow variant, if
+	// available.
+	Narrow Transformer = Transformer{narrowTransform{}}
+)
+
+// TODO: Consider the following options:
+// - Treat Ambiguous runes that have a halfwidth counterpart as wide, or some
+//   generalized variant of this.
+// - Consider a wide Won character to be the default width (or some generalized
+//   variant of this).
+// - Filter the set of characters that gets converted (the preferred approach is
+//   to allow applying filters to transforms).