compress/flate: add pure huffman deflater

Add a "HuffmanOnly" compression level, where the input is
only entropy encoded.

The output is fully inflate compatible. Typical compression
is reduction is about 50% of typical level 1 compression, however
the compression time is very stable, and does not vary as much as
nearly as much level 1 compression (or Snappy).

This mode is useful for:
 * HTTP compression in a CPU limited environment.
 * Entropy encoding Snappy compressed data, for archiving, etc.
 * Compression where compression time needs to be predictable.
 * Fast network transfer.

Snappy "usually" performs inbetween this and level 1 compression-wise,
but at the same speed as "Huffman", so this is not a replacement,
but a good supplement for Snappy, since it usually can compress
Snappy output further.

This is implemented as level -2, since this would be too much of a
compression reduction to replace level 1.

>go test -bench=Encode -cpu=1
BenchmarkEncodeDigitsHuffman1e4            30000             52334 ns/op         191.08 MB/s
BenchmarkEncodeDigitsHuffman1e5             3000            518343 ns/op         192.92 MB/s
BenchmarkEncodeDigitsHuffman1e6              300           5356884 ns/op         186.68 MB/s
BenchmarkEncodeDigitsSpeed1e4               5000            324214 ns/op          30.84 MB/s
BenchmarkEncodeDigitsSpeed1e5                500           3952614 ns/op          25.30 MB/s
BenchmarkEncodeDigitsSpeed1e6                 30          40760350 ns/op          24.53 MB/s
BenchmarkEncodeDigitsDefault1e4             5000            387056 ns/op          25.84 MB/s
BenchmarkEncodeDigitsDefault1e5              300           5950614 ns/op          16.80 MB/s
BenchmarkEncodeDigitsDefault1e6               20          63842195 ns/op          15.66 MB/s
BenchmarkEncodeDigitsCompress1e4            5000            391859 ns/op          25.52 MB/s
BenchmarkEncodeDigitsCompress1e5             300           5707112 ns/op          17.52 MB/s
BenchmarkEncodeDigitsCompress1e6              20          59839465 ns/op          16.71 MB/s
BenchmarkEncodeTwainHuffman1e4             20000             73498 ns/op         136.06 MB/s
BenchmarkEncodeTwainHuffman1e5              2000            595892 ns/op         167.82 MB/s
BenchmarkEncodeTwainHuffman1e6               200           6059016 ns/op         165.04 MB/s
BenchmarkEncodeTwainSpeed1e4                5000            321212 ns/op          31.13 MB/s
BenchmarkEncodeTwainSpeed1e5                 500           2823873 ns/op          35.41 MB/s
BenchmarkEncodeTwainSpeed1e6                  50          27237864 ns/op          36.71 MB/s
BenchmarkEncodeTwainDefault1e4              3000            454634 ns/op          22.00 MB/s
BenchmarkEncodeTwainDefault1e5               200           6859537 ns/op          14.58 MB/s
BenchmarkEncodeTwainDefault1e6                20          71547405 ns/op          13.98 MB/s
BenchmarkEncodeTwainCompress1e4             3000            462307 ns/op          21.63 MB/s
BenchmarkEncodeTwainCompress1e5              200           7534992 ns/op          13.27 MB/s
BenchmarkEncodeTwainCompress1e6               20          80353365 ns/op          12.45 MB/s
PASS
ok      compress/flate  55.333s

Change-Id: I8e12ad13220e50d4cf7ddba6f292333efad61b0c
Reviewed-on: https://go-review.googlesource.com/20982Reviewed-by: Joe Tsai <joetsai@digital-static.net>
Reviewed-by: Nigel Tao <nigeltao@golang.org>

src/compress/flate/deflate.go

@@ -15,6 +15,7 @@ const (
 	BestSpeed          = 1
 	BestCompression    = 9
 	DefaultCompression = -1
+	HuffmanOnly        = -2 // Disables match search and only does Huffman entropy reduction.
 	logWindowSize      = 15
 	windowSize         = 1 << logWindowSize
 	windowMask         = windowSize - 1
@@ -462,6 +463,18 @@ func (d *compressor) store() {
 	d.windowEnd = 0
 }
 
+// storeHuff compresses and stores the currently added data
+// when the d.window is full or we are at the end of the stream.
+// Any error that occurred will be in d.err
+func (d *compressor) storeHuff() {
+	if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
+		return
+	}
+	d.w.writeBlockHuff(false, d.window[:d.windowEnd])
+	d.err = d.w.err
+	d.windowEnd = 0
+}
+
 func (d *compressor) write(b []byte) (n int, err error) {
 	if d.err != nil {
 		return 0, d.err
@@ -500,6 +513,10 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
 		d.window = make([]byte, maxStoreBlockSize)
 		d.fill = (*compressor).fillStore
 		d.step = (*compressor).store
+	case level == HuffmanOnly:
+		d.window = make([]byte, maxStoreBlockSize)
+		d.fill = (*compressor).fillStore
+		d.step = (*compressor).storeHuff
 	case level == DefaultCompression:
 		level = 6
 		fallthrough
@@ -509,7 +526,7 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
 		d.fill = (*compressor).fillDeflate
 		d.step = (*compressor).deflate
 	default:
-		return fmt.Errorf("flate: invalid compression level %d: want value in range [-1, 9]", level)
+		return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
 	}
 	return nil
 }
@@ -565,10 +582,14 @@ func (d *compressor) close() error {
 // Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
 // higher levels typically run slower but compress more. Level 0
 // (NoCompression) does not attempt any compression; it only adds the
-// necessary DEFLATE framing. Level -1 (DefaultCompression) uses the default
-// compression level.
+// necessary DEFLATE framing.
+// Level -1 (DefaultCompression) uses the default compression level.
+// Level -2 (HuffmanOnly) will use Huffman compression only, giving
+// a very fast compression for all types of input, but sacrificing considerable
+// compression efficiency.
+//
 //
-// If level is in the range [-1, 9] then the error returned will be nil.
+// If level is in the range [-2, 9] then the error returned will be nil.
 // Otherwise the error returned will be non-nil.
 func NewWriter(w io.Writer, level int) (*Writer, error) {
 	var dw Writer

src/compress/flate/deflate_test.go

@@ -332,15 +332,17 @@ func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name str
 	testSync(t, level, input, name)
 }
 
-func testToFromWithLimit(t *testing.T, input []byte, name string, limit [10]int) {
+func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) {
 	for i := 0; i < 10; i++ {
 		testToFromWithLevelAndLimit(t, i, input, name, limit[i])
 	}
+	// Test HuffmanCompression
+	testToFromWithLevelAndLimit(t, -2, input, name, limit[10])
 }
 
 func TestDeflateInflate(t *testing.T) {
 	for i, h := range deflateInflateTests {
-		testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [10]int{})
+		testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{})
 	}
 }
 
@@ -356,19 +358,19 @@ func TestReverseBits(t *testing.T) {
 type deflateInflateStringTest struct {
 	filename string
 	label    string
-	limit    [10]int
+	limit    [11]int
 }
 
 var deflateInflateStringTests = []deflateInflateStringTest{
 	{
 		"../testdata/e.txt",
 		"2.718281828...",
-		[...]int{100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790},
+		[...]int{100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683},
 	},
 	{
 		"../testdata/Mark.Twain-Tom.Sawyer.txt",
 		"Mark.Twain-Tom.Sawyer",
-		[...]int{407330, 187598, 180361, 172974, 169160, 163476, 160936, 160506, 160295, 160295},
+		[...]int{407330, 187598, 180361, 172974, 169160, 163476, 160936, 160506, 160295, 160295, 233460},
 	},
 }
 

src/compress/flate/reader_test.go

@@ -74,8 +74,12 @@ const (
 	speed    = BestSpeed
 	default_ = DefaultCompression
 	compress = BestCompression
+	huffman  = HuffmanOnly
 )
 
+func BenchmarkDecodeDigitsHuffman1e4(b *testing.B)  { benchmarkDecode(b, digits, huffman, 1e4) }
+func BenchmarkDecodeDigitsHuffman1e5(b *testing.B)  { benchmarkDecode(b, digits, huffman, 1e5) }
+func BenchmarkDecodeDigitsHuffman1e6(b *testing.B)  { benchmarkDecode(b, digits, huffman, 1e6) }
 func BenchmarkDecodeDigitsSpeed1e4(b *testing.B)    { benchmarkDecode(b, digits, speed, 1e4) }
 func BenchmarkDecodeDigitsSpeed1e5(b *testing.B)    { benchmarkDecode(b, digits, speed, 1e5) }
 func BenchmarkDecodeDigitsSpeed1e6(b *testing.B)    { benchmarkDecode(b, digits, speed, 1e6) }
@@ -85,6 +89,9 @@ func BenchmarkDecodeDigitsDefault1e6(b *testing.B)  { benchmarkDecode(b, digits,
 func BenchmarkDecodeDigitsCompress1e4(b *testing.B) { benchmarkDecode(b, digits, compress, 1e4) }
 func BenchmarkDecodeDigitsCompress1e5(b *testing.B) { benchmarkDecode(b, digits, compress, 1e5) }
 func BenchmarkDecodeDigitsCompress1e6(b *testing.B) { benchmarkDecode(b, digits, compress, 1e6) }
+func BenchmarkDecodeTwainHuffman1e4(b *testing.B)   { benchmarkDecode(b, twain, huffman, 1e4) }
+func BenchmarkDecodeTwainHuffman1e5(b *testing.B)   { benchmarkDecode(b, twain, huffman, 1e5) }
+func BenchmarkDecodeTwainHuffman1e6(b *testing.B)   { benchmarkDecode(b, twain, huffman, 1e6) }
 func BenchmarkDecodeTwainSpeed1e4(b *testing.B)     { benchmarkDecode(b, twain, speed, 1e4) }
 func BenchmarkDecodeTwainSpeed1e5(b *testing.B)     { benchmarkDecode(b, twain, speed, 1e5) }
 func BenchmarkDecodeTwainSpeed1e6(b *testing.B)     { benchmarkDecode(b, twain, speed, 1e6) }

src/compress/flate/writer_test.go

@@ -45,6 +45,9 @@ func benchmarkEncoder(b *testing.B, testfile, level, n int) {
 	}
 }
 
+func BenchmarkEncodeDigitsHuffman1e4(b *testing.B)  { benchmarkEncoder(b, digits, huffman, 1e4) }
+func BenchmarkEncodeDigitsHuffman1e5(b *testing.B)  { benchmarkEncoder(b, digits, huffman, 1e5) }
+func BenchmarkEncodeDigitsHuffman1e6(b *testing.B)  { benchmarkEncoder(b, digits, huffman, 1e6) }
 func BenchmarkEncodeDigitsSpeed1e4(b *testing.B)    { benchmarkEncoder(b, digits, speed, 1e4) }
 func BenchmarkEncodeDigitsSpeed1e5(b *testing.B)    { benchmarkEncoder(b, digits, speed, 1e5) }
 func BenchmarkEncodeDigitsSpeed1e6(b *testing.B)    { benchmarkEncoder(b, digits, speed, 1e6) }
@@ -54,6 +57,9 @@ func BenchmarkEncodeDigitsDefault1e6(b *testing.B)  { benchmarkEncoder(b, digits
 func BenchmarkEncodeDigitsCompress1e4(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e4) }
 func BenchmarkEncodeDigitsCompress1e5(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e5) }
 func BenchmarkEncodeDigitsCompress1e6(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e6) }
+func BenchmarkEncodeTwainHuffman1e4(b *testing.B)   { benchmarkEncoder(b, twain, huffman, 1e4) }
+func BenchmarkEncodeTwainHuffman1e5(b *testing.B)   { benchmarkEncoder(b, twain, huffman, 1e5) }
+func BenchmarkEncodeTwainHuffman1e6(b *testing.B)   { benchmarkEncoder(b, twain, huffman, 1e6) }
 func BenchmarkEncodeTwainSpeed1e4(b *testing.B)     { benchmarkEncoder(b, twain, speed, 1e4) }
 func BenchmarkEncodeTwainSpeed1e5(b *testing.B)     { benchmarkEncoder(b, twain, speed, 1e5) }
 func BenchmarkEncodeTwainSpeed1e6(b *testing.B)     { benchmarkEncoder(b, twain, speed, 1e6) }
@@ -131,16 +137,17 @@ func TestWriteError(t *testing.T) {
 
 // Test if two runs produce identical results
 // even when writing different sizes to the Writer.
-func TestDeterministicL0(t *testing.T) { testDeterministic(0, t) }
-func TestDeterministicL1(t *testing.T) { testDeterministic(1, t) }
-func TestDeterministicL2(t *testing.T) { testDeterministic(2, t) }
-func TestDeterministicL3(t *testing.T) { testDeterministic(3, t) }
-func TestDeterministicL4(t *testing.T) { testDeterministic(4, t) }
-func TestDeterministicL5(t *testing.T) { testDeterministic(5, t) }
-func TestDeterministicL6(t *testing.T) { testDeterministic(6, t) }
-func TestDeterministicL7(t *testing.T) { testDeterministic(7, t) }
-func TestDeterministicL8(t *testing.T) { testDeterministic(8, t) }
-func TestDeterministicL9(t *testing.T) { testDeterministic(9, t) }
+func TestDeterministicL0(t *testing.T)  { testDeterministic(0, t) }
+func TestDeterministicL1(t *testing.T)  { testDeterministic(1, t) }
+func TestDeterministicL2(t *testing.T)  { testDeterministic(2, t) }
+func TestDeterministicL3(t *testing.T)  { testDeterministic(3, t) }
+func TestDeterministicL4(t *testing.T)  { testDeterministic(4, t) }
+func TestDeterministicL5(t *testing.T)  { testDeterministic(5, t) }
+func TestDeterministicL6(t *testing.T)  { testDeterministic(6, t) }
+func TestDeterministicL7(t *testing.T)  { testDeterministic(7, t) }
+func TestDeterministicL8(t *testing.T)  { testDeterministic(8, t) }
+func TestDeterministicL9(t *testing.T)  { testDeterministic(9, t) }
+func TestDeterministicLM2(t *testing.T) { testDeterministic(-2, t) }
 
 func testDeterministic(i int, t *testing.T) {
 	// Test so much we cross a good number of block boundaries.