Commit cc9ed447 authored by Nigel Tao's avatar Nigel Tao

compress: make flate, gzip and zlib's NewWriterXxx functions all return

(*Writer, error) if they take a compression level, and *Writer otherwise.
Rename gzip's Compressor and Decompressor to Writer and Reader, similar to
flate and zlib.

Clarify commentary when writing gzip metadata that is not representable
as Latin-1, and fix io.EOF comment bug.

Also refactor gzip_test to be more straightforward.

Fixes #2839.

R=rsc, r, rsc, bradfitz
CC=golang-dev
https://golang.org/cl/5639057
parent 0bc6836e
...@@ -878,6 +878,20 @@ If the argument size is too small or invalid, it is adjusted. ...@@ -878,6 +878,20 @@ If the argument size is too small or invalid, it is adjusted.
What little code is affected will be caught by the compiler and must be updated by hand. What little code is affected will be caught by the compiler and must be updated by hand.
</p> </p>
<h3 id="bufio">The compress/flate, compress/gzip and compress/zlib packages</h3>
<p>
In Go 1, the NewWriterXxx functions in compress/flate, compress/gzip and
compress/zlib all return (*Writer, error) if they take a compression level,
and *Writer otherwise. Package gzip's Compressor and Decompressor types have
been renamed to Writer and Reader.
</p>
<p>
<em>Updating</em>:
What little code is affected will be caught by the compiler and must be updated by hand.
</p>
<h3 id="crypto_elliptic">The crypto/elliptic package</h3> <h3 id="crypto_elliptic">The crypto/elliptic package</h3>
<p> <p>
......
...@@ -782,6 +782,20 @@ If the argument size is too small or invalid, it is adjusted. ...@@ -782,6 +782,20 @@ If the argument size is too small or invalid, it is adjusted.
What little code is affected will be caught by the compiler and must be updated by hand. What little code is affected will be caught by the compiler and must be updated by hand.
</p> </p>
<h3 id="bufio">The compress/flate, compress/gzip and compress/zlib packages</h3>
<p>
In Go 1, the NewWriterXxx functions in compress/flate, compress/gzip and
compress/zlib all return (*Writer, error) if they take a compression level,
and *Writer otherwise. Package gzip's Compressor and Decompressor types have
been renamed to Writer and Reader.
</p>
<p>
<em>Updating</em>:
What little code is affected will be caught by the compiler and must be updated by hand.
</p>
<h3 id="crypto_elliptic">The crypto/elliptic package</h3> <h3 id="crypto_elliptic">The crypto/elliptic package</h3>
<p> <p>
......
...@@ -127,7 +127,11 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) { ...@@ -127,7 +127,11 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
case Store: case Store:
fw.comp = nopCloser{fw.compCount} fw.comp = nopCloser{fw.compCount}
case Deflate: case Deflate:
fw.comp = flate.NewWriter(fw.compCount, 5) var err error
fw.comp, err = flate.NewWriter(fw.compCount, 5)
if err != nil {
return nil, err
}
default: default:
return nil, ErrAlgorithm return nil, ErrAlgorithm
} }
......
...@@ -408,17 +408,22 @@ func (d *compressor) close() error { ...@@ -408,17 +408,22 @@ func (d *compressor) close() error {
return d.w.err return d.w.err
} }
// NewWriter returns a new Writer compressing // NewWriter returns a new Writer compressing data at the given level.
// data at the given level. Following zlib, levels // Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
// range from 1 (BestSpeed) to 9 (BestCompression); // higher levels typically run slower but compress more. Level 0
// higher levels typically run slower but compress more. // (NoCompression) does not attempt any compression; it only adds the
// Level 0 (NoCompression) does not attempt any // necessary DEFLATE framing. Level -1 (DefaultCompression) uses the default
// compression; it only adds the necessary DEFLATE framing. // compression level.
func NewWriter(w io.Writer, level int) *Writer { //
// If level is in the range [-1, 9] then the error returned will be nil.
// Otherwise the error returned will be non-nil.
func NewWriter(w io.Writer, level int) (*Writer, error) {
const logWindowSize = logMaxOffsetSize const logWindowSize = logMaxOffsetSize
var dw Writer var dw Writer
dw.d.init(w, level) if err := dw.d.init(w, level); err != nil {
return &dw return nil, err
}
return &dw, nil
} }
// NewWriterDict is like NewWriter but initializes the new // NewWriterDict is like NewWriter but initializes the new
...@@ -427,13 +432,16 @@ func NewWriter(w io.Writer, level int) *Writer { ...@@ -427,13 +432,16 @@ func NewWriter(w io.Writer, level int) *Writer {
// any compressed output. The compressed data written to w // any compressed output. The compressed data written to w
// can only be decompressed by a Reader initialized with the // can only be decompressed by a Reader initialized with the
// same dictionary. // same dictionary.
func NewWriterDict(w io.Writer, level int, dict []byte) *Writer { func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
dw := &dictWriter{w, false} dw := &dictWriter{w, false}
zw := NewWriter(dw, level) zw, err := NewWriter(dw, level)
if err != nil {
return nil, err
}
zw.Write(dict) zw.Write(dict)
zw.Flush() zw.Flush()
dw.enabled = true dw.enabled = true
return zw return zw, err
} }
type dictWriter struct { type dictWriter struct {
......
...@@ -81,7 +81,11 @@ func largeDataChunk() []byte { ...@@ -81,7 +81,11 @@ func largeDataChunk() []byte {
func TestDeflate(t *testing.T) { func TestDeflate(t *testing.T) {
for _, h := range deflateTests { for _, h := range deflateTests {
var buf bytes.Buffer var buf bytes.Buffer
w := NewWriter(&buf, h.level) w, err := NewWriter(&buf, h.level)
if err != nil {
t.Errorf("NewWriter: %v", err)
continue
}
w.Write(h.in) w.Write(h.in)
w.Close() w.Close()
if !bytes.Equal(buf.Bytes(), h.out) { if !bytes.Equal(buf.Bytes(), h.out) {
...@@ -151,7 +155,11 @@ func testSync(t *testing.T, level int, input []byte, name string) { ...@@ -151,7 +155,11 @@ func testSync(t *testing.T, level int, input []byte, name string) {
buf := newSyncBuffer() buf := newSyncBuffer()
buf1 := new(bytes.Buffer) buf1 := new(bytes.Buffer)
buf.WriteMode() buf.WriteMode()
w := NewWriter(io.MultiWriter(buf, buf1), level) w, err := NewWriter(io.MultiWriter(buf, buf1), level)
if err != nil {
t.Errorf("NewWriter: %v", err)
return
}
r := NewReader(buf) r := NewReader(buf)
// Write half the input and read back. // Write half the input and read back.
...@@ -213,7 +221,7 @@ func testSync(t *testing.T, level int, input []byte, name string) { ...@@ -213,7 +221,7 @@ func testSync(t *testing.T, level int, input []byte, name string) {
// stream should work for ordinary reader too // stream should work for ordinary reader too
r = NewReader(buf1) r = NewReader(buf1)
out, err := ioutil.ReadAll(r) out, err = ioutil.ReadAll(r)
if err != nil { if err != nil {
t.Errorf("testSync: read: %s", err) t.Errorf("testSync: read: %s", err)
return return
...@@ -224,31 +232,31 @@ func testSync(t *testing.T, level int, input []byte, name string) { ...@@ -224,31 +232,31 @@ func testSync(t *testing.T, level int, input []byte, name string) {
} }
} }
func testToFromWithLevel(t *testing.T, level int, input []byte, name string) error { func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) {
return testToFromWithLevelAndLimit(t, level, input, name, -1)
}
func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) error {
var buffer bytes.Buffer var buffer bytes.Buffer
w := NewWriter(&buffer, level) w, err := NewWriter(&buffer, level)
if err != nil {
t.Errorf("NewWriter: %v", err)
return
}
w.Write(input) w.Write(input)
w.Close() w.Close()
if limit > 0 && buffer.Len() > limit { if limit > 0 && buffer.Len() > limit {
t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit) t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit)
return
} }
r := NewReader(&buffer) r := NewReader(&buffer)
out, err := ioutil.ReadAll(r) out, err := ioutil.ReadAll(r)
if err != nil { if err != nil {
t.Errorf("read: %s", err) t.Errorf("read: %s", err)
return err return
} }
r.Close() r.Close()
if !bytes.Equal(input, out) { if !bytes.Equal(input, out) {
t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name)
return
} }
testSync(t, level, input, name) testSync(t, level, input, name)
return nil
} }
func testToFromWithLimit(t *testing.T, input []byte, name string, limit [10]int) { func testToFromWithLimit(t *testing.T, input []byte, name string, limit [10]int) {
...@@ -257,13 +265,9 @@ func testToFromWithLimit(t *testing.T, input []byte, name string, limit [10]int) ...@@ -257,13 +265,9 @@ func testToFromWithLimit(t *testing.T, input []byte, name string, limit [10]int)
} }
} }
func testToFrom(t *testing.T, input []byte, name string) {
testToFromWithLimit(t, input, name, [10]int{})
}
func TestDeflateInflate(t *testing.T) { func TestDeflateInflate(t *testing.T) {
for i, h := range deflateInflateTests { for i, h := range deflateInflateTests {
testToFrom(t, h.in, fmt.Sprintf("#%d", i)) testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [10]int{})
} }
} }
...@@ -311,7 +315,10 @@ func TestReaderDict(t *testing.T) { ...@@ -311,7 +315,10 @@ func TestReaderDict(t *testing.T) {
text = "hello again world" text = "hello again world"
) )
var b bytes.Buffer var b bytes.Buffer
w := NewWriter(&b, 5) w, err := NewWriter(&b, 5)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
w.Write([]byte(dict)) w.Write([]byte(dict))
w.Flush() w.Flush()
b.Reset() b.Reset()
...@@ -334,7 +341,10 @@ func TestWriterDict(t *testing.T) { ...@@ -334,7 +341,10 @@ func TestWriterDict(t *testing.T) {
text = "hello again world" text = "hello again world"
) )
var b bytes.Buffer var b bytes.Buffer
w := NewWriter(&b, 5) w, err := NewWriter(&b, 5)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
w.Write([]byte(dict)) w.Write([]byte(dict))
w.Flush() w.Flush()
b.Reset() b.Reset()
...@@ -342,7 +352,7 @@ func TestWriterDict(t *testing.T) { ...@@ -342,7 +352,7 @@ func TestWriterDict(t *testing.T) {
w.Close() w.Close()
var b1 bytes.Buffer var b1 bytes.Buffer
w = NewWriterDict(&b1, 5, []byte(dict)) w, _ = NewWriterDict(&b1, 5, []byte(dict))
w.Write([]byte(text)) w.Write([]byte(text))
w.Close() w.Close()
...@@ -353,7 +363,10 @@ func TestWriterDict(t *testing.T) { ...@@ -353,7 +363,10 @@ func TestWriterDict(t *testing.T) {
// See http://code.google.com/p/go/issues/detail?id=2508 // See http://code.google.com/p/go/issues/detail?id=2508
func TestRegression2508(t *testing.T) { func TestRegression2508(t *testing.T) {
w := NewWriter(ioutil.Discard, 1) w, err := NewWriter(ioutil.Discard, 1)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
buf := make([]byte, 1024) buf := make([]byte, 1024)
for i := 0; i < 131072; i++ { for i := 0; i < 131072; i++ {
if _, err := w.Write(buf); err != nil { if _, err := w.Write(buf); err != nil {
......
...@@ -16,9 +16,6 @@ import ( ...@@ -16,9 +16,6 @@ import (
"time" "time"
) )
// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of
// the 0x00-0x7f range to ISO 8859-1 (Latin-1).
const ( const (
gzipID1 = 0x1f gzipID1 = 0x1f
gzipID2 = 0x8b gzipID2 = 0x8b
...@@ -41,7 +38,7 @@ var ErrHeader = errors.New("invalid gzip header") ...@@ -41,7 +38,7 @@ var ErrHeader = errors.New("invalid gzip header")
var ErrChecksum = errors.New("gzip checksum error") var ErrChecksum = errors.New("gzip checksum error")
// The gzip file stores a header giving metadata about the compressed file. // The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Compressor and Decompressor structs. // That header is exposed as the fields of the Writer and Reader structs.
type Header struct { type Header struct {
Comment string // comment Comment string // comment
Extra []byte // "extra data" Extra []byte // "extra data"
...@@ -50,21 +47,21 @@ type Header struct { ...@@ -50,21 +47,21 @@ type Header struct {
OS byte // operating system type OS byte // operating system type
} }
// An Decompressor is an io.Reader that can be read to retrieve // A Reader is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file. // uncompressed data from a gzip-format compressed file.
// //
// In general, a gzip file can be a concatenation of gzip files, // In general, a gzip file can be a concatenation of gzip files,
// each with its own header. Reads from the Decompressor // each with its own header. Reads from the Reader
// return the concatenation of the uncompressed data of each. // return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Decompressor fields. // Only the first header is recorded in the Reader fields.
// //
// Gzip files store a length and checksum of the uncompressed data. // Gzip files store a length and checksum of the uncompressed data.
// The Decompressor will return a ErrChecksum when Read // The Reader will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not // reaches the end of the uncompressed data if it does not
// have the expected length or checksum. Clients should treat data // have the expected length or checksum. Clients should treat data
// returned by Read as tentative until they receive the successful // returned by Read as tentative until they receive the io.EOF
// (zero length, nil error) Read marking the end of the data. // marking the end of the data.
type Decompressor struct { type Reader struct {
Header Header
r flate.Reader r flate.Reader
decompressor io.ReadCloser decompressor io.ReadCloser
...@@ -75,11 +72,11 @@ type Decompressor struct { ...@@ -75,11 +72,11 @@ type Decompressor struct {
err error err error
} }
// NewReader creates a new Decompressor reading the given reader. // NewReader creates a new Reader reading the given reader.
// The implementation buffers input and may read more data than necessary from r. // The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Decompressor when done. // It is the caller's responsibility to call Close on the Reader when done.
func NewReader(r io.Reader) (*Decompressor, error) { func NewReader(r io.Reader) (*Reader, error) {
z := new(Decompressor) z := new(Reader)
z.r = makeReader(r) z.r = makeReader(r)
z.digest = crc32.NewIEEE() z.digest = crc32.NewIEEE()
if err := z.readHeader(true); err != nil { if err := z.readHeader(true); err != nil {
...@@ -93,7 +90,7 @@ func get4(p []byte) uint32 { ...@@ -93,7 +90,7 @@ func get4(p []byte) uint32 {
return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
} }
func (z *Decompressor) readString() (string, error) { func (z *Reader) readString() (string, error) {
var err error var err error
needconv := false needconv := false
for i := 0; ; i++ { for i := 0; ; i++ {
...@@ -122,7 +119,7 @@ func (z *Decompressor) readString() (string, error) { ...@@ -122,7 +119,7 @@ func (z *Decompressor) readString() (string, error) {
panic("not reached") panic("not reached")
} }
func (z *Decompressor) read2() (uint32, error) { func (z *Reader) read2() (uint32, error) {
_, err := io.ReadFull(z.r, z.buf[0:2]) _, err := io.ReadFull(z.r, z.buf[0:2])
if err != nil { if err != nil {
return 0, err return 0, err
...@@ -130,7 +127,7 @@ func (z *Decompressor) read2() (uint32, error) { ...@@ -130,7 +127,7 @@ func (z *Decompressor) read2() (uint32, error) {
return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
} }
func (z *Decompressor) readHeader(save bool) error { func (z *Reader) readHeader(save bool) error {
_, err := io.ReadFull(z.r, z.buf[0:10]) _, err := io.ReadFull(z.r, z.buf[0:10])
if err != nil { if err != nil {
return err return err
...@@ -196,7 +193,7 @@ func (z *Decompressor) readHeader(save bool) error { ...@@ -196,7 +193,7 @@ func (z *Decompressor) readHeader(save bool) error {
return nil return nil
} }
func (z *Decompressor) Read(p []byte) (n int, err error) { func (z *Reader) Read(p []byte) (n int, err error) {
if z.err != nil { if z.err != nil {
return 0, z.err return 0, z.err
} }
...@@ -236,5 +233,5 @@ func (z *Decompressor) Read(p []byte) (n int, err error) { ...@@ -236,5 +233,5 @@ func (z *Decompressor) Read(p []byte) (n int, err error) {
return z.Read(p) return z.Read(p)
} }
// Calling Close does not close the wrapped io.Reader originally passed to NewReader. // Close closes the Reader. It does not close the underlying io.Reader.
func (z *Decompressor) Close() error { return z.decompressor.Close() } func (z *Reader) Close() error { return z.decompressor.Close() }
...@@ -7,6 +7,7 @@ package gzip ...@@ -7,6 +7,7 @@ package gzip
import ( import (
"compress/flate" "compress/flate"
"errors" "errors"
"fmt"
"hash" "hash"
"hash/crc32" "hash/crc32"
"io" "io"
...@@ -21,9 +22,9 @@ const ( ...@@ -21,9 +22,9 @@ const (
DefaultCompression = flate.DefaultCompression DefaultCompression = flate.DefaultCompression
) )
// A Compressor is an io.WriteCloser that satisfies writes by compressing data written // A Writer is an io.WriteCloser that satisfies writes by compressing data written
// to its wrapped io.Writer. // to its wrapped io.Writer.
type Compressor struct { type Writer struct {
Header Header
w io.Writer w io.Writer
level int level int
...@@ -35,25 +36,40 @@ type Compressor struct { ...@@ -35,25 +36,40 @@ type Compressor struct {
err error err error
} }
// NewWriter calls NewWriterLevel with the default compression level. // NewWriter creates a new Writer that satisfies writes by compressing data
func NewWriter(w io.Writer) (*Compressor, error) { // written to w.
return NewWriterLevel(w, DefaultCompression) //
// It is the caller's responsibility to call Close on the WriteCloser when done.
// Writes may be buffered and not flushed until Close.
//
// Callers that wish to set the fields in Writer.Header must do so before
// the first call to Write or Close. The Comment and Name header fields are
// UTF-8 strings in Go, but the underlying format requires NUL-terminated ISO
// 8859-1 (Latin-1). NUL or non-Latin-1 runes in those strings will lead to an
// error on Write.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevel(w, DefaultCompression)
return z
} }
// NewWriterLevel creates a new Compressor writing to the given writer. // NewWriterLevel is like NewWriter but specifies the compression level instead
// Writes may be buffered and not flushed until Close. // of assuming DefaultCompression.
// Callers that wish to set the fields in Compressor.Header must //
// do so before the first call to Write or Close. // The compression level can be DefaultCompression, NoCompression, or any
// It is the caller's responsibility to call Close on the WriteCloser when done. // integer value between BestSpeed and BestCompression inclusive. The error
// level is the compression level, which can be DefaultCompression, NoCompression, // returned will be nil if the level is valid.
// or any integer value between BestSpeed and BestCompression (inclusive). func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
func NewWriterLevel(w io.Writer, level int) (*Compressor, error) { if level < DefaultCompression || level > BestCompression {
z := new(Compressor) return nil, fmt.Errorf("gzip: invalid compression level: %d", level)
z.OS = 255 // unknown }
z.w = w return &Writer{
z.level = level Header: Header{
z.digest = crc32.NewIEEE() OS: 255, // unknown
return z, nil },
w: w,
level: level,
digest: crc32.NewIEEE(),
}, nil
} }
// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). // GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
...@@ -70,7 +86,7 @@ func put4(p []byte, v uint32) { ...@@ -70,7 +86,7 @@ func put4(p []byte, v uint32) {
} }
// writeBytes writes a length-prefixed byte slice to z.w. // writeBytes writes a length-prefixed byte slice to z.w.
func (z *Compressor) writeBytes(b []byte) error { func (z *Writer) writeBytes(b []byte) error {
if len(b) > 0xffff { if len(b) > 0xffff {
return errors.New("gzip.Write: Extra data is too large") return errors.New("gzip.Write: Extra data is too large")
} }
...@@ -83,10 +99,10 @@ func (z *Compressor) writeBytes(b []byte) error { ...@@ -83,10 +99,10 @@ func (z *Compressor) writeBytes(b []byte) error {
return err return err
} }
// writeString writes a string (in ISO 8859-1 (Latin-1) format) to z.w. // writeString writes a UTF-8 string s in GZIP's format to z.w.
func (z *Compressor) writeString(s string) error { // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). func (z *Writer) writeString(s string) (err error) {
var err error // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII.
needconv := false needconv := false
for _, v := range s { for _, v := range s {
if v == 0 || v > 0xff { if v == 0 || v > 0xff {
...@@ -114,7 +130,7 @@ func (z *Compressor) writeString(s string) error { ...@@ -114,7 +130,7 @@ func (z *Compressor) writeString(s string) error {
return err return err
} }
func (z *Compressor) Write(p []byte) (int, error) { func (z *Writer) Write(p []byte) (int, error) {
if z.err != nil { if z.err != nil {
return 0, z.err return 0, z.err
} }
...@@ -165,7 +181,7 @@ func (z *Compressor) Write(p []byte) (int, error) { ...@@ -165,7 +181,7 @@ func (z *Compressor) Write(p []byte) (int, error) {
return n, z.err return n, z.err
} }
} }
z.compressor = flate.NewWriter(z.w, z.level) z.compressor, _ = flate.NewWriter(z.w, z.level)
} }
z.size += uint32(len(p)) z.size += uint32(len(p))
z.digest.Write(p) z.digest.Write(p)
...@@ -173,8 +189,8 @@ func (z *Compressor) Write(p []byte) (int, error) { ...@@ -173,8 +189,8 @@ func (z *Compressor) Write(p []byte) (int, error) {
return n, z.err return n, z.err
} }
// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. // Close closes the Writer. It does not close the underlying io.Writer.
func (z *Compressor) Close() error { func (z *Writer) Close() error {
if z.err != nil { if z.err != nil {
return z.err return z.err
} }
......
...@@ -7,108 +7,153 @@ package gzip ...@@ -7,108 +7,153 @@ package gzip
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"io"
"io/ioutil" "io/ioutil"
"testing" "testing"
"time" "time"
) )
// pipe creates two ends of a pipe that gzip and gunzip, and runs dfunc at the // TestEmpty tests that an empty payload still forms a valid GZIP stream.
// writer end and cfunc at the reader end. func TestEmpty(t *testing.T) {
func pipe(t *testing.T, dfunc func(*Compressor), cfunc func(*Decompressor)) { buf := new(bytes.Buffer)
piper, pipew := io.Pipe()
defer piper.Close() if err := NewWriter(buf).Close(); err != nil {
go func() { t.Fatalf("Writer.Close: %v", err)
defer pipew.Close() }
compressor, err := NewWriter(pipew)
if err != nil { r, err := NewReader(buf)
t.Fatalf("%v", err) if err != nil {
} t.Fatalf("NewReader: %v", err)
defer compressor.Close() }
dfunc(compressor) b, err := ioutil.ReadAll(r)
}()
decompressor, err := NewReader(piper)
if err != nil { if err != nil {
t.Fatalf("%v", err) t.Fatalf("ReadAll: %v", err)
}
if len(b) != 0 {
t.Fatalf("got %d bytes, want 0", len(b))
}
if err := r.Close(); err != nil {
t.Fatalf("Reader.Close: %v", err)
} }
defer decompressor.Close()
cfunc(decompressor)
} }
// Tests that an empty payload still forms a valid GZIP stream. // TestRoundTrip tests that gzipping and then gunzipping is the identity
func TestEmpty(t *testing.T) { // function.
pipe(t, func TestRoundTrip(t *testing.T) {
func(compressor *Compressor) {}, buf := new(bytes.Buffer)
func(decompressor *Decompressor) {
b, err := ioutil.ReadAll(decompressor) w := NewWriter(buf)
if err != nil { w.Comment = "comment"
t.Fatalf("%v", err) w.Extra = []byte("extra")
} w.ModTime = time.Unix(1e8, 0)
if len(b) != 0 { w.Name = "name"
t.Fatalf("did not read an empty slice") if _, err := w.Write([]byte("payload")); err != nil {
} t.Fatalf("Write: %v", err)
}) }
} if err := w.Close(); err != nil {
t.Fatalf("Writer.Close: %v", err)
}
// Tests that gzipping and then gunzipping is the identity function. r, err := NewReader(buf)
func TestWriter(t *testing.T) { if err != nil {
pipe(t, t.Fatalf("NewReader: %v", err)
func(compressor *Compressor) { }
compressor.Comment = "Äußerung" b, err := ioutil.ReadAll(r)
//compressor.Comment = "comment" if err != nil {
compressor.Extra = []byte("extra") t.Fatalf("ReadAll: %v", err)
compressor.ModTime = time.Unix(1e8, 0) }
compressor.Name = "name" if string(b) != "payload" {
_, err := compressor.Write([]byte("payload")) t.Fatalf("payload is %q, want %q", string(b), "payload")
if err != nil { }
t.Fatalf("%v", err) if r.Comment != "comment" {
} t.Fatalf("comment is %q, want %q", r.Comment, "comment")
}, }
func(decompressor *Decompressor) { if string(r.Extra) != "extra" {
b, err := ioutil.ReadAll(decompressor) t.Fatalf("extra is %q, want %q", r.Extra, "extra")
if err != nil { }
t.Fatalf("%v", err) if r.ModTime.Unix() != 1e8 {
} t.Fatalf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8))
if string(b) != "payload" { }
t.Fatalf("payload is %q, want %q", string(b), "payload") if r.Name != "name" {
} t.Fatalf("name is %q, want %q", r.Name, "name")
if decompressor.Comment != "Äußerung" { }
t.Fatalf("comment is %q, want %q", decompressor.Comment, "Äußerung") if err := r.Close(); err != nil {
} t.Fatalf("Reader.Close: %v", err)
if string(decompressor.Extra) != "extra" { }
t.Fatalf("extra is %q, want %q", decompressor.Extra, "extra")
}
if decompressor.ModTime.Unix() != 1e8 {
t.Fatalf("mtime is %d, want %d", decompressor.ModTime.Unix(), uint32(1e8))
}
if decompressor.Name != "name" {
t.Fatalf("name is %q, want %q", decompressor.Name, "name")
}
})
} }
// TestLatin1 tests the internal functions for converting to and from Latin-1.
func TestLatin1(t *testing.T) { func TestLatin1(t *testing.T) {
latin1 := []byte{0xc4, 'u', 0xdf, 'e', 'r', 'u', 'n', 'g', 0} latin1 := []byte{0xc4, 'u', 0xdf, 'e', 'r', 'u', 'n', 'g', 0}
utf8 := "Äußerung" utf8 := "Äußerung"
z := Decompressor{r: bufio.NewReader(bytes.NewBuffer(latin1))} z := Reader{r: bufio.NewReader(bytes.NewBuffer(latin1))}
s, err := z.readString() s, err := z.readString()
if err != nil { if err != nil {
t.Fatalf("%v", err) t.Fatalf("readString: %v", err)
} }
if s != utf8 { if s != utf8 {
t.Fatalf("string is %q, want %q", s, utf8) t.Fatalf("read latin-1: got %q, want %q", s, utf8)
} }
buf := bytes.NewBuffer(make([]byte, 0, len(latin1))) buf := bytes.NewBuffer(make([]byte, 0, len(latin1)))
c := Compressor{w: buf} c := Writer{w: buf}
if err = c.writeString(utf8); err != nil { if err = c.writeString(utf8); err != nil {
t.Fatalf("%v", err) t.Fatalf("writeString: %v", err)
} }
s = buf.String() s = buf.String()
if s != string(latin1) { if s != string(latin1) {
t.Fatalf("string is %v, want %v", s, latin1) t.Fatalf("write utf-8: got %q, want %q", s, string(latin1))
}
}
// TestLatin1RoundTrip tests that metadata that is representable in Latin-1
// survives a round trip.
func TestLatin1RoundTrip(t *testing.T) {
testCases := []struct {
name string
ok bool
}{
{"", true},
{"ASCII is OK", true},
{"unless it contains a NUL\x00", false},
{"no matter where \x00 occurs", false},
{"\x00\x00\x00", false},
{"Látin-1 also passes (U+00E1)", true},
{"but LĀtin Extended-A (U+0100) does not", false},
{"neither does 日本語", false},
{"invalid UTF-8 also \xffails", false},
{"\x00 as does Látin-1 with NUL", false},
}
for _, tc := range testCases {
buf := new(bytes.Buffer)
w := NewWriter(buf)
w.Name = tc.name
err := w.Close()
if (err == nil) != tc.ok {
t.Errorf("Writer.Close: name = %q, err = %v", tc.name, err)
continue
}
if !tc.ok {
continue
}
r, err := NewReader(buf)
if err != nil {
t.Errorf("NewReader: %v", err)
continue
}
_, err = ioutil.ReadAll(r)
if err != nil {
t.Errorf("ReadAll: %v", err)
continue
}
if r.Name != tc.name {
t.Errorf("name is %q, want %q", r.Name, tc.name)
continue
}
if err := r.Close(); err != nil {
t.Errorf("Reader.Close: %v", err)
continue
}
} }
//if s, err = buf.ReadString(0); err != nil {
//t.Fatalf("%v", err)
//}
} }
...@@ -6,7 +6,7 @@ package zlib ...@@ -6,7 +6,7 @@ package zlib
import ( import (
"compress/flate" "compress/flate"
"errors" "fmt"
"hash" "hash"
"hash/adler32" "hash/adler32"
"io" "io"
...@@ -24,30 +24,55 @@ const ( ...@@ -24,30 +24,55 @@ const (
// A Writer takes data written to it and writes the compressed // A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see NewWriter). // form of that data to an underlying writer (see NewWriter).
type Writer struct { type Writer struct {
w io.Writer w io.Writer
compressor *flate.Writer level int
digest hash.Hash32 dict []byte
err error compressor *flate.Writer
scratch [4]byte digest hash.Hash32
err error
scratch [4]byte
wroteHeader bool
} }
// NewWriter calls NewWriterLevel with the default compression level. // NewWriter creates a new Writer that satisfies writes by compressing data
func NewWriter(w io.Writer) (*Writer, error) { // written to w.
return NewWriterLevel(w, DefaultCompression) //
// It is the caller's responsibility to call Close on the WriteCloser when done.
// Writes may be buffered and not flushed until Close.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevelDict(w, DefaultCompression, nil)
return z
} }
// NewWriterLevel calls NewWriterDict with no dictionary. // NewWriterLevel is like NewWriter but specifies the compression level instead
// of assuming DefaultCompression.
//
// The compression level can be DefaultCompression, NoCompression, or any
// integer value between BestSpeed and BestCompression inclusive. The error
// returned will be nil if the level is valid.
func NewWriterLevel(w io.Writer, level int) (*Writer, error) { func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
return NewWriterDict(w, level, nil) return NewWriterLevelDict(w, level, nil)
} }
// NewWriterDict creates a new io.WriteCloser that satisfies writes by compressing data written to w. // NewWriterLevelDict is like NewWriterLevel but specifies a dictionary to
// It is the caller's responsibility to call Close on the WriteCloser when done. // compress with.
// level is the compression level, which can be DefaultCompression, NoCompression, //
// or any integer value between BestSpeed and BestCompression (inclusive). // The dictionary may be nil. If not, its contents should not be modified until
// dict is the preset dictionary to compress with, or nil to use no dictionary. // the Writer is closed.
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) {
z := new(Writer) if level < DefaultCompression || level > BestCompression {
return nil, fmt.Errorf("zlib: invalid compression level: %d", level)
}
return &Writer{
w: w,
level: level,
dict: dict,
}, nil
}
// writeHeader writes the ZLIB header.
func (z *Writer) writeHeader() (err error) {
z.wroteHeader = true
// ZLIB has a two-byte header (as documented in RFC 1950). // ZLIB has a two-byte header (as documented in RFC 1950).
// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
// The next four bits is the CM (compression method), which is 8 for deflate. // The next four bits is the CM (compression method), which is 8 for deflate.
...@@ -56,7 +81,7 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { ...@@ -56,7 +81,7 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
// 0=fastest, 1=fast, 2=default, 3=best. // 0=fastest, 1=fast, 2=default, 3=best.
// The next bit, FDICT, is set if a dictionary is given. // The next bit, FDICT, is set if a dictionary is given.
// The final five FCHECK bits form a mod-31 checksum. // The final five FCHECK bits form a mod-31 checksum.
switch level { switch z.level {
case 0, 1: case 0, 1:
z.scratch[1] = 0 << 6 z.scratch[1] = 0 << 6
case 2, 3, 4, 5: case 2, 3, 4, 5:
...@@ -66,35 +91,38 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { ...@@ -66,35 +91,38 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
case 7, 8, 9: case 7, 8, 9:
z.scratch[1] = 3 << 6 z.scratch[1] = 3 << 6
default: default:
return nil, errors.New("level out of range") panic("unreachable")
} }
if dict != nil { if z.dict != nil {
z.scratch[1] |= 1 << 5 z.scratch[1] |= 1 << 5
} }
z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31)
_, err := w.Write(z.scratch[0:2]) if _, err = z.w.Write(z.scratch[0:2]); err != nil {
if err != nil { return err
return nil, err
} }
if dict != nil { if z.dict != nil {
// The next four bytes are the Adler-32 checksum of the dictionary. // The next four bytes are the Adler-32 checksum of the dictionary.
checksum := adler32.Checksum(dict) checksum := adler32.Checksum(z.dict)
z.scratch[0] = uint8(checksum >> 24) z.scratch[0] = uint8(checksum >> 24)
z.scratch[1] = uint8(checksum >> 16) z.scratch[1] = uint8(checksum >> 16)
z.scratch[2] = uint8(checksum >> 8) z.scratch[2] = uint8(checksum >> 8)
z.scratch[3] = uint8(checksum >> 0) z.scratch[3] = uint8(checksum >> 0)
_, err = w.Write(z.scratch[0:4]) if _, err = z.w.Write(z.scratch[0:4]); err != nil {
if err != nil { return err
return nil, err
} }
} }
z.w = w z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict)
z.compressor = flate.NewWriterDict(w, level, dict) if err != nil {
return err
}
z.digest = adler32.New() z.digest = adler32.New()
return z, nil return nil
} }
func (z *Writer) Write(p []byte) (n int, err error) { func (z *Writer) Write(p []byte) (n int, err error) {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil { if z.err != nil {
return 0, z.err return 0, z.err
} }
...@@ -112,6 +140,9 @@ func (z *Writer) Write(p []byte) (n int, err error) { ...@@ -112,6 +140,9 @@ func (z *Writer) Write(p []byte) (n int, err error) {
// Flush flushes the underlying compressor. // Flush flushes the underlying compressor.
func (z *Writer) Flush() error { func (z *Writer) Flush() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil { if z.err != nil {
return z.err return z.err
} }
...@@ -121,6 +152,9 @@ func (z *Writer) Flush() error { ...@@ -121,6 +152,9 @@ func (z *Writer) Flush() error {
// Calling Close does not close the wrapped io.Writer originally passed to NewWriter. // Calling Close does not close the wrapped io.Writer originally passed to NewWriter.
func (z *Writer) Close() error { func (z *Writer) Close() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil { if z.err != nil {
return z.err return z.err
} }
......
...@@ -52,7 +52,7 @@ func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { ...@@ -52,7 +52,7 @@ func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) {
defer piper.Close() defer piper.Close()
go func() { go func() {
defer pipew.Close() defer pipew.Close()
zlibw, err := NewWriterDict(pipew, level, dict) zlibw, err := NewWriterLevelDict(pipew, level, dict)
if err != nil { if err != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
return return
...@@ -125,9 +125,9 @@ func TestWriterDict(t *testing.T) { ...@@ -125,9 +125,9 @@ func TestWriterDict(t *testing.T) {
func TestWriterDictIsUsed(t *testing.T) { func TestWriterDictIsUsed(t *testing.T) {
var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.")
var buf bytes.Buffer var buf bytes.Buffer
compressor, err := NewWriterDict(&buf, BestCompression, input) compressor, err := NewWriterLevelDict(&buf, BestCompression, input)
if err != nil { if err != nil {
t.Errorf("error in NewWriterDict: %s", err) t.Errorf("error in NewWriterLevelDict: %s", err)
return return
} }
compressor.Write(input) compressor.Write(input)
......
...@@ -22,7 +22,7 @@ func TestRead(t *testing.T) { ...@@ -22,7 +22,7 @@ func TestRead(t *testing.T) {
} }
var z bytes.Buffer var z bytes.Buffer
f := flate.NewWriter(&z, 5) f, _ := flate.NewWriter(&z, 5)
f.Write(b) f.Write(b)
f.Close() f.Close()
if z.Len() < len(b)*99/100 { if z.Len() < len(b)*99/100 {
......
...@@ -263,10 +263,7 @@ func filter(cr *[nFilter][]byte, pr []byte, bpp int) int { ...@@ -263,10 +263,7 @@ func filter(cr *[nFilter][]byte, pr []byte, bpp int) int {
} }
func writeImage(w io.Writer, m image.Image, cb int) error { func writeImage(w io.Writer, m image.Image, cb int) error {
zw, err := zlib.NewWriter(w) zw := zlib.NewWriter(w)
if err != nil {
return err
}
defer zw.Close() defer zw.Close()
bpp := 0 // Bytes per pixel. bpp := 0 // Bytes per pixel.
...@@ -391,8 +388,7 @@ func writeImage(w io.Writer, m image.Image, cb int) error { ...@@ -391,8 +388,7 @@ func writeImage(w io.Writer, m image.Image, cb int) error {
f := filter(&cr, pr, bpp) f := filter(&cr, pr, bpp)
// Write the compressed bytes. // Write the compressed bytes.
_, err = zw.Write(cr[f]) if _, err := zw.Write(cr[f]); err != nil {
if err != nil {
return err return err
} }
......
...@@ -321,9 +321,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) { ...@@ -321,9 +321,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) {
} }
if test.compressed { if test.compressed {
buf.WriteString("Content-Encoding: gzip\r\n") buf.WriteString("Content-Encoding: gzip\r\n")
var err error wr = gzip.NewWriter(wr)
wr, err = gzip.NewWriter(wr)
checkErr(err, "gzip.NewWriter")
} }
buf.WriteString("\r\n") buf.WriteString("\r\n")
...@@ -337,7 +335,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) { ...@@ -337,7 +335,7 @@ func TestReadResponseCloseInMiddle(t *testing.T) {
wr.Write(chunk) wr.Write(chunk)
} }
if test.compressed { if test.compressed {
err := wr.(*gzip.Compressor).Close() err := wr.(*gzip.Writer).Close()
checkErr(err, "compressor close") checkErr(err, "compressor close")
} }
if test.chunked { if test.chunked {
......
...@@ -441,11 +441,7 @@ func TestRoundTripGzip(t *testing.T) { ...@@ -441,11 +441,7 @@ func TestRoundTripGzip(t *testing.T) {
} }
if accept == "gzip" { if accept == "gzip" {
rw.Header().Set("Content-Encoding", "gzip") rw.Header().Set("Content-Encoding", "gzip")
gz, err := gzip.NewWriter(rw) gz := gzip.NewWriter(rw)
if err != nil {
t.Errorf("gzip NewWriter: %v", err)
return
}
gz.Write([]byte(responseBody)) gz.Write([]byte(responseBody))
gz.Close() gz.Close()
} else { } else {
...@@ -512,7 +508,7 @@ func TestTransportGzip(t *testing.T) { ...@@ -512,7 +508,7 @@ func TestTransportGzip(t *testing.T) {
rw.Header().Set("Content-Length", strconv.Itoa(buf.Len())) rw.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
}() }()
} }
gz, _ := gzip.NewWriter(w) gz := gzip.NewWriter(w)
gz.Write([]byte(testString)) gz.Write([]byte(testString))
if req.FormValue("body") == "large" { if req.FormValue("body") == "large" {
io.CopyN(gz, rand.Reader, nRandBytes) io.CopyN(gz, rand.Reader, nRandBytes)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment