diff --git a/vendor/github.com/andybalholm/brotli/README.md b/vendor/github.com/andybalholm/brotli/README.md index 02d115e55a307..1ea7fdb759d3b 100644 --- a/vendor/github.com/andybalholm/brotli/README.md +++ b/vendor/github.com/andybalholm/brotli/README.md @@ -3,3 +3,5 @@ It was translated from the reference implementation (https://github.com/google/b with the `c2go` tool at https://github.com/andybalholm/c2go. I am using it in production with https://github.com/andybalholm/redwood. + +API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc. diff --git a/vendor/github.com/andybalholm/brotli/backward_references.go b/vendor/github.com/andybalholm/brotli/backward_references.go index 1642e471ad09f..008c054d1c03e 100644 --- a/vendor/github.com/andybalholm/brotli/backward_references.go +++ b/vendor/github.com/andybalholm/brotli/backward_references.go @@ -1,5 +1,9 @@ package brotli +import ( + "sync" +) + /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -31,13 +35,10 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin return distance + numDistanceShortCodes - 1 } -/* "commands" points to the next output command to write to, "*num_commands" is - initially the total amount of commands output by previous - CreateBackwardReferences calls, and must be incremented by the amount written - by this call. */ -func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +var hasherSearchResultPool sync.Pool + +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) - var orig_commands []command = commands var insert_length uint = *last_insert_len var pos_end uint = position + num_bytes var store_end uint @@ -57,8 +58,14 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, /* Minimum score to accept a backward reference. */ hasher.PrepareDistanceCache(dist_cache) - var sr2 hasherSearchResult - var sr hasherSearchResult + sr2, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr2 == nil { + sr2 = &hasherSearchResult{} + } + sr, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr == nil { + sr = &hasherSearchResult{} + } for position+hasher.HashTypeLength() < pos_end { var max_length uint = pos_end - position @@ -67,7 +74,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, sr.len_code_delta = 0 sr.distance = 0 sr.score = kMinScore - hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, sr) if sr.score > kMinScore { /* Found a match. Let's look for something even better ahead. */ var delayed_backward_references_in_row int = 0 @@ -83,14 +90,14 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, sr2.distance = 0 sr2.score = kMinScore max_distance = brotli_min_size_t(position+1, max_backward_limit) - hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, sr2) if sr2.score >= sr.score+cost_diff_lazy { /* Ok, let's just write one byte for now and start a match from the next byte. */ position++ insert_length++ - sr = sr2 + *sr = *sr2 delayed_backward_references_in_row++ if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { continue @@ -114,8 +121,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, hasher.PrepareDistanceCache(dist_cache) } - initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code) - commands = commands[1:] + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)) } *num_literals += insert_length @@ -173,5 +179,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, insert_length += pos_end - position *last_insert_len = insert_length - *num_commands += uint(-cap(commands) + cap(orig_commands)) + + hasherSearchResultPool.Put(sr) + hasherSearchResultPool.Put(sr2) } diff --git a/vendor/github.com/andybalholm/brotli/backward_references_hq.go b/vendor/github.com/andybalholm/brotli/backward_references_hq.go index 5eac736133f2c..21629c1cdb718 100644 --- a/vendor/github.com/andybalholm/brotli/backward_references_hq.go +++ b/vendor/github.com/andybalholm/brotli/backward_references_hq.go @@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co } } -func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) { var histogram_literal [numLiteralSymbols]uint32 var histogram_cmd [numCommandSymbols]uint32 var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 var cost_literal [numLiteralSymbols]float32 var pos uint = position - last_insert_len var min_cost_cmd float32 = kInfinity - var i uint var cost_cmd []float32 = self.cost_cmd_[:] var literal_costs []float32 @@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu histogram_cmd = [numCommandSymbols]uint32{} histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} - for i = 0; i < num_commands; i++ { + for i := range commands { var inslength uint = uint(commands[i].insert_len_) var copylength uint = uint(commandCopyLen(&commands[i])) var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF @@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) - for i = 0; i < numCommandSymbols; i++ { + for i := 0; i < numCommandSymbols; i++ { min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) } @@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu { literal_costs = self.literal_costs_ var literal_carry float32 = 0.0 - var num_bytes uint = self.num_bytes_ + num_bytes := int(self.num_bytes_) literal_costs[0] = 0.0 - for i = 0; i < num_bytes; i++ { - literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] + for i := 0; i < num_bytes; i++ { + literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]] literal_costs[i+1] = literal_costs[i] + literal_carry literal_carry -= literal_costs[i+1] - literal_costs[i] } @@ -502,7 +501,9 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) if cost < nodes[pos+len].u.cost { updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) - result = brotli_max_size_t(result, uint(len)) + if len > result { + result = len + } } } } @@ -530,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { } /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ -func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var pos uint = 0 var offset uint32 = nodes[0].u.next @@ -552,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) var is_dictionary bool = (distance > max_distance+gap) var dist_code uint = uint(zopfliNodeDistanceCode(next)) - initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)) if !is_dictionary && dist_code > 0 { dist_cache[3] = dist_cache[2] @@ -679,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, return computeShortestPathFromNodes(num_bytes, nodes) } -func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var nodes []zopfliNode nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliNodes(nodes, num_bytes+1) - *num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) + zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) nodes = nil } -func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var num_matches []uint32 = make([]uint32, num_bytes) var matches_size uint = 4 * num_bytes @@ -703,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer var orig_num_literals uint var orig_last_insert_len uint var orig_dist_cache [4]int - var orig_num_commands uint + var orig_num_commands int var model zopfliCostModel var nodes []zopfliNode var matches []backwardMatch = make([]backwardMatch, matches_size) @@ -769,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer orig_num_literals = *num_literals orig_last_insert_len = *last_insert_len copy(orig_dist_cache[:], dist_cache[:4]) - orig_num_commands = *num_commands + orig_num_commands = len(*commands) nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliCostModel(&model, ¶ms.dist, num_bytes) for i = 0; i < 2; i++ { @@ -777,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer if i == 0 { zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) } else { - zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len) } - *num_commands = orig_num_commands + *commands = (*commands)[:orig_num_commands] *num_literals = orig_num_literals *last_insert_len = orig_last_insert_len copy(dist_cache, orig_dist_cache[:4]) - *num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) + zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) } diff --git a/vendor/github.com/andybalholm/brotli/block_splitter.go b/vendor/github.com/andybalholm/brotli/block_splitter.go index 2ccff45a3ead2..978a1314748db 100644 --- a/vendor/github.com/andybalholm/brotli/block_splitter.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter.go @@ -33,23 +33,21 @@ const ( kMinItersForRefining uint = 100 ) -func countLiterals(cmds []command, num_commands uint) uint { +func countLiterals(cmds []command) uint { var total_length uint = 0 /* Count how many we have. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { total_length += uint(cmds[i].insert_len_) } return total_length } -func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { +func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) { var pos uint = 0 var from_pos uint = offset & mask - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var insert_len uint = uint(cmds[i].insert_len_) if from_pos+insert_len > mask { var head_size uint = mask + 1 - from_pos @@ -90,24 +88,19 @@ const clustersPerBatch = 16 func initBlockSplit(self *blockSplit) { self.num_types = 0 self.num_blocks = 0 - self.types = nil - self.lengths = nil + self.types = self.types[:0] + self.lengths = self.lengths[:0] self.types_alloc_size = 0 self.lengths_alloc_size = 0 } -func destroyBlockSplit(self *blockSplit) { - self.types = nil - self.lengths = nil -} - -func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { +func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { { - var literals_count uint = countLiterals(cmds, num_commands) + var literals_count uint = countLiterals(cmds) var literals []byte = make([]byte, literals_count) /* Create a continuous array of literals. */ - copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) + copyLiteralsToByteArray(cmds, data, pos, mask, literals) /* Create the block split on the array of literals. Literal histograms have alphabet size 256. */ @@ -116,28 +109,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u literals = nil } { - var insert_and_copy_codes []uint16 = make([]uint16, num_commands) + var insert_and_copy_codes []uint16 = make([]uint16, len(cmds)) /* Compute prefix codes for commands. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { insert_and_copy_codes[i] = cmds[i].cmd_prefix_ } /* Create the block split on the array of command prefixes. */ - splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) + splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) /* TODO: reuse for distances? */ insert_and_copy_codes = nil } { - var distance_prefixes []uint16 = make([]uint16, num_commands) + var distance_prefixes []uint16 = make([]uint16, len(cmds)) var j uint = 0 /* Create a continuous array of distance prefixes. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_command.go b/vendor/github.com/andybalholm/brotli/block_splitter_command.go index e505fe13e732d..9dec13e4d907c 100644 --- a/vendor/github.com/andybalholm/brotli/block_splitter_command.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter_command.go @@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids histogram_symbols = nil } -func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { +func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + length := uint(len(data)) var data_size uint = histogramDataSizeCommand() var num_histograms uint = length/literals_per_histogram + 1 var histograms []histogramCommand diff --git a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go index 395f604904327..2470f84e4b08b 100644 --- a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go +++ b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go @@ -1,6 +1,9 @@ package brotli -import "math" +import ( + "math" + "sync" +) const maxHuffmanTreeSize = (2*numCommandSymbols + 1) @@ -118,7 +121,7 @@ func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) { *bits = uint64(length) - 1 } -func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { +func storeCommandExtra(cmd *command, bw *bitWriter) { var copylen_code uint32 = commandCopyLenCode(cmd) var inscode uint16 = getInsertLengthCode(uint(cmd.insert_len_)) var copycode uint16 = getCopyLengthCode(uint(copylen_code)) @@ -126,7 +129,7 @@ func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { var insextraval uint64 = uint64(cmd.insert_len_) - uint64(getInsertBase(inscode)) var copyextraval uint64 = uint64(copylen_code) - uint64(getCopyBase(copycode)) var bits uint64 = copyextraval< 0 REQUIRES: length <= (1 << 24) */ -func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix *uint, storage []byte) { +func storeCompressedMetaBlockHeader(is_final_block bool, length uint, bw *bitWriter) { var lenbits uint64 var nlenbits uint var nibblesbits uint64 @@ -166,41 +169,41 @@ func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix } /* Write ISLAST bit. */ - writeBits(1, is_final, storage_ix, storage) + bw.writeBits(1, is_final) /* Write ISEMPTY bit. */ if is_final_block { - writeBits(1, 0, storage_ix, storage) + bw.writeBits(1, 0) } encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) - writeBits(2, nibblesbits, storage_ix, storage) - writeBits(nlenbits, lenbits, storage_ix, storage) + bw.writeBits(2, nibblesbits) + bw.writeBits(nlenbits, lenbits) if !is_final_block { /* Write ISUNCOMPRESSED bit. */ - writeBits(1, 0, storage_ix, storage) + bw.writeBits(1, 0) } } /* Stores the uncompressed meta-block header. REQUIRES: length > 0 REQUIRES: length <= (1 << 24) */ -func storeUncompressedMetaBlockHeader(length uint, storage_ix *uint, storage []byte) { +func storeUncompressedMetaBlockHeader(length uint, bw *bitWriter) { var lenbits uint64 var nlenbits uint var nibblesbits uint64 /* Write ISLAST bit. Uncompressed block cannot be the last one, so set to 0. */ - writeBits(1, 0, storage_ix, storage) + bw.writeBits(1, 0) encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) - writeBits(2, nibblesbits, storage_ix, storage) - writeBits(nlenbits, lenbits, storage_ix, storage) + bw.writeBits(2, nibblesbits) + bw.writeBits(nlenbits, lenbits) /* Write ISUNCOMPRESSED bit. */ - writeBits(1, 1, storage_ix, storage) + bw.writeBits(1, 1) } var storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder = [codeLengthCodes]byte{1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15} @@ -208,7 +211,7 @@ var storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder = [codeLengthCodes]byte var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols = [6]byte{0, 7, 3, 2, 1, 15} var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths = [6]byte{2, 4, 3, 2, 2, 4} -func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth []byte, storage_ix *uint, storage []byte) { +func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth []byte, bw *bitWriter) { var skip_some uint = 0 var codes_to_store uint = codeLengthCodes /* The bit lengths of the Huffman code over the code length alphabet @@ -238,38 +241,38 @@ func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth } } - writeBits(2, uint64(skip_some), storage_ix, storage) + bw.writeBits(2, uint64(skip_some)) { var i uint for i = skip_some; i < codes_to_store; i++ { var l uint = uint(code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[i]]) - writeBits(uint(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths[l]), uint64(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols[l]), storage_ix, storage) + bw.writeBits(uint(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths[l]), uint64(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols[l])) } } } -func storeHuffmanTreeToBitMask(huffman_tree_size uint, huffman_tree []byte, huffman_tree_extra_bits []byte, code_length_bitdepth []byte, code_length_bitdepth_symbols []uint16, storage_ix *uint, storage []byte) { +func storeHuffmanTreeToBitMask(huffman_tree_size uint, huffman_tree []byte, huffman_tree_extra_bits []byte, code_length_bitdepth []byte, code_length_bitdepth_symbols []uint16, bw *bitWriter) { var i uint for i = 0; i < huffman_tree_size; i++ { var ix uint = uint(huffman_tree[i]) - writeBits(uint(code_length_bitdepth[ix]), uint64(code_length_bitdepth_symbols[ix]), storage_ix, storage) + bw.writeBits(uint(code_length_bitdepth[ix]), uint64(code_length_bitdepth_symbols[ix])) /* Extra bits */ switch ix { case repeatPreviousCodeLength: - writeBits(2, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) + bw.writeBits(2, uint64(huffman_tree_extra_bits[i])) case repeatZeroCodeLength: - writeBits(3, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) + bw.writeBits(3, uint64(huffman_tree_extra_bits[i])) } } } -func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max_bits uint, storage_ix *uint, storage []byte) { +func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max_bits uint, bw *bitWriter) { /* value of 1 indicates a simple Huffman code */ - writeBits(2, 1, storage_ix, storage) + bw.writeBits(2, 1) - writeBits(2, uint64(num_symbols)-1, storage_ix, storage) /* NSYM - 1 */ + bw.writeBits(2, uint64(num_symbols)-1) /* NSYM - 1 */ { /* Sort */ var i uint @@ -286,17 +289,17 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max } if num_symbols == 2 { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) } else if num_symbols == 3 { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) } else { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + bw.writeBits(max_bits, uint64(symbols[3])) /* tree-select */ var tmp int @@ -305,13 +308,13 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max } else { tmp = 0 } - writeBits(1, uint64(tmp), storage_ix, storage) + bw.writeBits(1, uint64(tmp)) } } /* num = alphabet size depths = symbol depths */ -func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *uint, storage []byte) { +func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, bw *bitWriter) { var huffman_tree [numCommandSymbols]byte var huffman_tree_extra_bits [numCommandSymbols]byte var huffman_tree_size uint = 0 @@ -354,19 +357,19 @@ func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *u convertBitDepthsToSymbols(code_length_bitdepth[:], codeLengthCodes, code_length_bitdepth_symbols[:]) /* Now, we have all the data, let's start storing it */ - storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], storage_ix, storage) + storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], bw) if num_codes == 1 { code_length_bitdepth[code] = 0 } /* Store the real Huffman tree now. */ - storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], storage_ix, storage) + storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], bw) } /* Builds a Huffman tree from histogram[0:length] into depth[0:length] and bits[0:length] and stores the encoded tree to the bit stream. */ -func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { +func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, bw *bitWriter) { var count uint = 0 var s4 = [4]uint{0} var i uint @@ -391,8 +394,8 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe } if count <= 1 { - writeBits(4, 1, storage_ix, storage) - writeBits(max_bits, uint64(s4[0]), storage_ix, storage) + bw.writeBits(4, 1) + bw.writeBits(max_bits, uint64(s4[0])) depth[s4[0]] = 0 bits[s4[0]] = 0 return @@ -405,17 +408,19 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe convertBitDepthsToSymbols(depth, histogram_length, bits) if count <= 4 { - storeSimpleHuffmanTree(depth, s4[:], count, max_bits, storage_ix, storage) + storeSimpleHuffmanTree(depth, s4[:], count, max_bits, bw) } else { - storeHuffmanTree(depth, histogram_length, tree, storage_ix, storage) + storeHuffmanTree(depth, histogram_length, tree, bw) } } -func sortHuffmanTree1(v0 *huffmanTree, v1 *huffmanTree) bool { +func sortHuffmanTree1(v0 huffmanTree, v1 huffmanTree) bool { return v0.total_count_ < v1.total_count_ } -func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { +var huffmanTreePool sync.Pool + +func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, bw *bitWriter) { var count uint = 0 var symbols = [4]uint{0} var length uint = 0 @@ -434,8 +439,8 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } if count <= 1 { - writeBits(4, 1, storage_ix, storage) - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + bw.writeBits(4, 1) + bw.writeBits(max_bits, uint64(symbols[0])) depth[symbols[0]] = 0 bits[symbols[0]] = 0 return @@ -446,7 +451,13 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } { var max_tree_size uint = 2*length + 1 - var tree []huffmanTree = make([]huffmanTree, max_tree_size) + tree, _ := huffmanTreePool.Get().(*[]huffmanTree) + if tree == nil || cap(*tree) < int(max_tree_size) { + tmp := make([]huffmanTree, max_tree_size) + tree = &tmp + } else { + *tree = (*tree)[:max_tree_size] + } var count_limit uint32 for count_limit = 1; ; count_limit *= 2 { var node int = 0 @@ -455,9 +466,9 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ l-- if histogram[l] != 0 { if histogram[l] >= count_limit { - initHuffmanTree(&tree[node:][0], histogram[l], -1, int16(l)) + initHuffmanTree(&(*tree)[node:][0], histogram[l], -1, int16(l)) } else { - initHuffmanTree(&tree[node:][0], count_limit, -1, int16(l)) + initHuffmanTree(&(*tree)[node:][0], count_limit, -1, int16(l)) } node++ @@ -471,7 +482,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ var j int = n + 1 var k int - sortHuffmanTreeItems(tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) + sortHuffmanTreeItems(*tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) /* The nodes are: [0, n): the sorted leaf nodes that we start with. @@ -482,15 +493,15 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ There will be (2n+1) elements at the end. */ initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) - tree[node] = sentinel + (*tree)[node] = sentinel node++ - tree[node] = sentinel + (*tree)[node] = sentinel node++ for k = n - 1; k > 0; k-- { var left int var right int - if tree[i].total_count_ <= tree[j].total_count_ { + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { left = i i++ } else { @@ -498,7 +509,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ j++ } - if tree[i].total_count_ <= tree[j].total_count_ { + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { right = i i++ } else { @@ -507,17 +518,17 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } /* The sentinel node becomes the parent node. */ - tree[node-1].total_count_ = tree[left].total_count_ + tree[right].total_count_ + (*tree)[node-1].total_count_ = (*tree)[left].total_count_ + (*tree)[right].total_count_ - tree[node-1].index_left_ = int16(left) - tree[node-1].index_right_or_value_ = int16(right) + (*tree)[node-1].index_left_ = int16(left) + (*tree)[node-1].index_right_or_value_ = int16(right) /* Add back the last sentinel node. */ - tree[node] = sentinel + (*tree)[node] = sentinel node++ } - if setDepth(2*n-1, tree, depth, 14) { + if setDepth(2*n-1, *tree, depth, 14) { /* We need to pack the Huffman tree in 14 bits. If this was not successful, add fake entities to the lowest values and retry. */ break @@ -525,7 +536,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } } - tree = nil + huffmanTreePool.Put(tree) } convertBitDepthsToSymbols(depth, length, bits) @@ -533,9 +544,9 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ var i uint /* value of 1 indicates a simple Huffman code */ - writeBits(2, 1, storage_ix, storage) + bw.writeBits(2, 1) - writeBits(2, uint64(count)-1, storage_ix, storage) /* NSYM - 1 */ + bw.writeBits(2, uint64(count)-1) /* NSYM - 1 */ /* Sort */ for i = 0; i < count; i++ { @@ -550,33 +561,27 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } if count == 2 { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) } else if count == 3 { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) } else { - writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) - writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + bw.writeBits(max_bits, uint64(symbols[3])) /* tree-select */ - var tmp int - if depth[symbols[0]] == 1 { - tmp = 1 - } else { - tmp = 0 - } - writeBits(1, uint64(tmp), storage_ix, storage) + bw.writeSingleBit(depth[symbols[0]] == 1) } } else { var previous_value byte = 8 var i uint /* Complex Huffman Tree */ - storeStaticCodeLengthCode(storage_ix, storage) + storeStaticCodeLengthCode(bw) /* Actual RLE coding. */ for i = 0; i < length; { @@ -589,21 +594,21 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ i += reps if value == 0 { - writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps], storage_ix, storage) + bw.writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps]) } else { if previous_value != value { - writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) reps-- } if reps < 3 { for reps != 0 { reps-- - writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) } } else { reps -= 3 - writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps], storage_ix, storage) + bw.writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps]) } previous_value = value @@ -728,7 +733,7 @@ const symbolBits = 9 var encodeContextMap_kSymbolMask uint32 = (1 << symbolBits) - 1 -func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []huffmanTree, storage_ix *uint, storage []byte) { +func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []huffmanTree, bw *bitWriter) { var i uint var rle_symbols []uint32 var max_run_length_prefix uint32 = 6 @@ -737,7 +742,7 @@ func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters var depths [maxContextMapSymbols]byte var bits [maxContextMapSymbols]uint16 - storeVarLenUint8(num_clusters-1, storage_ix, storage) + storeVarLenUint8(num_clusters-1, bw) if num_clusters == 1 { return @@ -752,45 +757,45 @@ func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters } { var use_rle bool = (max_run_length_prefix > 0) - writeSingleBit(use_rle, storage_ix, storage) + bw.writeSingleBit(use_rle) if use_rle { - writeBits(4, uint64(max_run_length_prefix)-1, storage_ix, storage) + bw.writeBits(4, uint64(max_run_length_prefix)-1) } } - buildAndStoreHuffmanTree(histogram[:], uint(uint32(num_clusters)+max_run_length_prefix), uint(uint32(num_clusters)+max_run_length_prefix), tree, depths[:], bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(histogram[:], uint(uint32(num_clusters)+max_run_length_prefix), uint(uint32(num_clusters)+max_run_length_prefix), tree, depths[:], bits[:], bw) for i = 0; i < num_rle_symbols; i++ { var rle_symbol uint32 = rle_symbols[i] & encodeContextMap_kSymbolMask var extra_bits_val uint32 = rle_symbols[i] >> symbolBits - writeBits(uint(depths[rle_symbol]), uint64(bits[rle_symbol]), storage_ix, storage) + bw.writeBits(uint(depths[rle_symbol]), uint64(bits[rle_symbol])) if rle_symbol > 0 && rle_symbol <= max_run_length_prefix { - writeBits(uint(rle_symbol), uint64(extra_bits_val), storage_ix, storage) + bw.writeBits(uint(rle_symbol), uint64(extra_bits_val)) } } - writeBits(1, 1, storage_ix, storage) /* use move-to-front */ + bw.writeBits(1, 1) /* use move-to-front */ rle_symbols = nil } /* Stores the block switch command with index block_ix to the bit stream. */ -func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, is_first_block bool, storage_ix *uint, storage []byte) { +func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, is_first_block bool, bw *bitWriter) { var typecode uint = nextBlockTypeCode(&code.type_code_calculator, block_type) var lencode uint var len_nextra uint32 var len_extra uint32 if !is_first_block { - writeBits(uint(code.type_depths[typecode]), uint64(code.type_bits[typecode]), storage_ix, storage) + bw.writeBits(uint(code.type_depths[typecode]), uint64(code.type_bits[typecode])) } getBlockLengthPrefixCode(block_len, &lencode, &len_nextra, &len_extra) - writeBits(uint(code.length_depths[lencode]), uint64(code.length_bits[lencode]), storage_ix, storage) - writeBits(uint(len_nextra), uint64(len_extra), storage_ix, storage) + bw.writeBits(uint(code.length_depths[lencode]), uint64(code.length_bits[lencode])) + bw.writeBits(uint(len_nextra), uint64(len_extra)) } /* Builds a BlockSplitCode data structure from the block split given by the vector of block types and block lengths and stores it to the bit stream. */ -func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) { +func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, bw *bitWriter) { var type_histo [maxBlockTypeSymbols]uint32 var length_histo [numBlockLenSymbols]uint32 var i uint @@ -808,17 +813,17 @@ func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint length_histo[blockLengthPrefixCode(lengths[i])]++ } - storeVarLenUint8(num_types-1, storage_ix, storage) + storeVarLenUint8(num_types-1, bw) if num_types > 1 { /* TODO: else? could StoreBlockSwitch occur? */ - buildAndStoreHuffmanTree(type_histo[0:], num_types+2, num_types+2, tree, code.type_depths[0:], code.type_bits[0:], storage_ix, storage) - buildAndStoreHuffmanTree(length_histo[0:], numBlockLenSymbols, numBlockLenSymbols, tree, code.length_depths[0:], code.length_bits[0:], storage_ix, storage) - storeBlockSwitch(code, lengths[0], types[0], true, storage_ix, storage) + buildAndStoreHuffmanTree(type_histo[0:], num_types+2, num_types+2, tree, code.type_depths[0:], code.type_bits[0:], bw) + buildAndStoreHuffmanTree(length_histo[0:], numBlockLenSymbols, numBlockLenSymbols, tree, code.length_depths[0:], code.length_bits[0:], bw) + storeBlockSwitch(code, lengths[0], types[0], true, bw) } } /* Stores a context map where the histogram type is always the block type. */ -func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, storage_ix *uint, storage []byte) { - storeVarLenUint8(num_types-1, storage_ix, storage) +func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, bw *bitWriter) { + storeVarLenUint8(num_types-1, bw) if num_types > 1 { var repeat_code uint = context_bits - 1 var repeat_bits uint = (1 << repeat_code) - 1 @@ -832,16 +837,16 @@ func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTre } /* Write RLEMAX. */ - writeBits(1, 1, storage_ix, storage) + bw.writeBits(1, 1) - writeBits(4, uint64(repeat_code)-1, storage_ix, storage) + bw.writeBits(4, uint64(repeat_code)-1) histogram[repeat_code] = uint32(num_types) histogram[0] = 1 for i = context_bits; i < alphabet_size; i++ { histogram[i] = 1 } - buildAndStoreHuffmanTree(histogram[:], alphabet_size, alphabet_size, tree, depths[:], bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(histogram[:], alphabet_size, alphabet_size, tree, depths[:], bits[:], bw) for i = 0; i < num_types; i++ { var tmp uint if i == 0 { @@ -850,13 +855,13 @@ func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTre tmp = i + context_bits - 1 } var code uint = tmp - writeBits(uint(depths[code]), uint64(bits[code]), storage_ix, storage) - writeBits(uint(depths[repeat_code]), uint64(bits[repeat_code]), storage_ix, storage) - writeBits(repeat_code, uint64(repeat_bits), storage_ix, storage) + bw.writeBits(uint(depths[code]), uint64(bits[code])) + bw.writeBits(uint(depths[repeat_code]), uint64(bits[repeat_code])) + bw.writeBits(repeat_code, uint64(repeat_bits)) } /* Write IMTF (inverse-move-to-front) bit. */ - writeBits(1, 1, storage_ix, storage) + bw.writeBits(1, 1) } } @@ -875,38 +880,48 @@ type blockEncoder struct { bits_ []uint16 } -func initBlockEncoder(self *blockEncoder, histogram_length uint, num_block_types uint, block_types []byte, block_lengths []uint32, num_blocks uint) { +var blockEncoderPool sync.Pool + +func getBlockEncoder(histogram_length uint, num_block_types uint, block_types []byte, block_lengths []uint32, num_blocks uint) *blockEncoder { + self, _ := blockEncoderPool.Get().(*blockEncoder) + + if self != nil { + self.block_ix_ = 0 + self.entropy_ix_ = 0 + self.depths_ = self.depths_[:0] + self.bits_ = self.bits_[:0] + } else { + self = &blockEncoder{} + } + self.histogram_length_ = histogram_length self.num_block_types_ = num_block_types self.block_types_ = block_types self.block_lengths_ = block_lengths self.num_blocks_ = num_blocks initBlockTypeCodeCalculator(&self.block_split_code_.type_code_calculator) - self.block_ix_ = 0 if num_blocks == 0 { self.block_len_ = 0 } else { self.block_len_ = uint(block_lengths[0]) } - self.entropy_ix_ = 0 - self.depths_ = nil - self.bits_ = nil + + return self } func cleanupBlockEncoder(self *blockEncoder) { - self.depths_ = nil - self.bits_ = nil + blockEncoderPool.Put(self) } /* Creates entropy codes of block lengths and block types and stores them to the bit stream. */ -func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, storage_ix *uint, storage []byte) { - buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, storage_ix, storage) +func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, bw *bitWriter) { + buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, bw) } /* Stores the next symbol with the entropy code of the current block type. Updates the block type and block length at block boundaries. */ -func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []byte) { +func storeSymbol(self *blockEncoder, symbol uint, bw *bitWriter) { if self.block_len_ == 0 { self.block_ix_++ var block_ix uint = self.block_ix_ @@ -914,20 +929,20 @@ func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []by var block_type byte = self.block_types_[block_ix] self.block_len_ = uint(block_len) self.entropy_ix_ = uint(block_type) * self.histogram_length_ - storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, bw) } self.block_len_-- { var ix uint = self.entropy_ix_ + symbol - writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) + bw.writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix])) } } /* Stores the next symbol with the entropy code of the current block type and context value. Updates the block type and block length at block boundaries. */ -func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, storage_ix *uint, storage []byte, context_bits uint) { +func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, bw *bitWriter, context_bits uint) { if self.block_len_ == 0 { self.block_ix_++ var block_ix uint = self.block_ix_ @@ -935,119 +950,134 @@ func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, conte var block_type byte = self.block_types_[block_ix] self.block_len_ = uint(block_len) self.entropy_ix_ = uint(block_type) << context_bits - storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, bw) } self.block_len_-- { var histo_ix uint = uint(context_map[self.entropy_ix_+context]) var ix uint = histo_ix*self.histogram_length_ + symbol - writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) + bw.writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix])) } } -func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { +func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) } } } -func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { +func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) } } } -func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { +func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { var table_size uint = histograms_size * self.histogram_length_ - self.depths_ = make([]byte, table_size) - self.bits_ = make([]uint16, table_size) + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } { var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) } } } -func jumpToByteBoundary(storage_ix *uint, storage []byte) { - *storage_ix = (*storage_ix + 7) &^ 7 - storage[*storage_ix>>3] = 0 -} - -func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) { +func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, bw *bitWriter) { var pos uint = start_pos var i uint var num_distance_symbols uint32 = params.dist.alphabet_size var num_effective_distance_symbols uint32 = num_distance_symbols var tree []huffmanTree var literal_context_lut contextLUT = getContextLUT(literal_context_mode) - var literal_enc blockEncoder - var command_enc blockEncoder - var distance_enc blockEncoder var dist *distanceParams = ¶ms.dist if params.large_window && num_effective_distance_symbols > numHistogramDistanceSymbols { num_effective_distance_symbols = numHistogramDistanceSymbols } - storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + storeCompressedMetaBlockHeader(is_last, length, bw) tree = make([]huffmanTree, maxHuffmanTreeSize) - initBlockEncoder(&literal_enc, numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) - initBlockEncoder(&command_enc, numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) - initBlockEncoder(&distance_enc, uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) + literal_enc := getBlockEncoder(numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) + command_enc := getBlockEncoder(numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) + distance_enc := getBlockEncoder(uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) - buildAndStoreBlockSwitchEntropyCodes(&literal_enc, tree, storage_ix, storage) - buildAndStoreBlockSwitchEntropyCodes(&command_enc, tree, storage_ix, storage) - buildAndStoreBlockSwitchEntropyCodes(&distance_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, bw) + buildAndStoreBlockSwitchEntropyCodes(command_enc, tree, bw) + buildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, bw) - writeBits(2, uint64(dist.distance_postfix_bits), storage_ix, storage) - writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits, storage_ix, storage) + bw.writeBits(2, uint64(dist.distance_postfix_bits)) + bw.writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits) for i = 0; i < mb.literal_split.num_types; i++ { - writeBits(2, uint64(literal_context_mode), storage_ix, storage) + bw.writeBits(2, uint64(literal_context_mode)) } if mb.literal_context_map_size == 0 { - storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, storage_ix, storage) + storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, bw) } else { - encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, storage_ix, storage) + encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, bw) } if mb.distance_context_map_size == 0 { - storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, storage_ix, storage) + storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, bw) } else { - encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, storage_ix, storage) + encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, bw) } - buildAndStoreEntropyCodesLiteral(&literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, storage_ix, storage) - buildAndStoreEntropyCodesCommand(&command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, storage_ix, storage) - buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) + buildAndStoreEntropyCodesLiteral(literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, bw) + buildAndStoreEntropyCodesCommand(command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, bw) + buildAndStoreEntropyCodesDistance(distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, bw) tree = nil - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) - storeSymbol(&command_enc, cmd_code, storage_ix, storage) - storeCommandExtra(&cmd, storage_ix, storage) + storeSymbol(command_enc, cmd_code, bw) + storeCommandExtra(&cmd, bw) if mb.literal_context_map_size == 0 { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { - storeSymbol(&literal_enc, uint(input[pos&mask]), storage_ix, storage) + storeSymbol(literal_enc, uint(input[pos&mask]), bw) pos++ } } else { @@ -1055,7 +1085,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b for j = uint(cmd.insert_len_); j != 0; j-- { var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut)) var literal byte = input[pos&mask] - storeSymbolWithContext(&literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits) + storeSymbolWithContext(literal_enc, uint(literal), context, mb.literal_context_map, bw, literalContextBits) prev_byte2 = prev_byte prev_byte = literal pos++ @@ -1071,30 +1101,28 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 var distextra uint64 = uint64(cmd.dist_extra_) if mb.distance_context_map_size == 0 { - storeSymbol(&distance_enc, dist_code, storage_ix, storage) + storeSymbol(distance_enc, dist_code, bw) } else { var context uint = uint(commandDistanceContext(&cmd)) - storeSymbolWithContext(&distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits) + storeSymbolWithContext(distance_enc, dist_code, context, mb.distance_context_map, bw, distanceContextBits) } - writeBits(uint(distnumextra), distextra, storage_ix, storage) + bw.writeBits(uint(distnumextra), distextra) } } } - cleanupBlockEncoder(&distance_enc) - cleanupBlockEncoder(&command_enc) - cleanupBlockEncoder(&literal_enc) + cleanupBlockEncoder(distance_enc) + cleanupBlockEncoder(command_enc) + cleanupBlockEncoder(literal_enc) if is_last { - jumpToByteBoundary(storage_ix, storage) + bw.jumpToByteBoundary() } } -func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { +func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -1109,18 +1137,16 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command } } -func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { +func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, bw *bitWriter) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) var j uint - writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) - storeCommandExtra(&cmd, storage_ix, storage) + bw.writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code])) + storeCommandExtra(&cmd, bw) for j = uint(cmd.insert_len_); j != 0; j-- { var literal byte = input[pos&mask] - writeBits(uint(lit_depth[literal]), uint64(lit_bits[literal]), storage_ix, storage) + bw.writeBits(uint(lit_depth[literal]), uint64(lit_bits[literal])) pos++ } @@ -1129,13 +1155,13 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands var dist_code uint = uint(cmd.dist_prefix_) & 0x3FF var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 var distextra uint32 = cmd.dist_extra_ - writeBits(uint(dist_depth[dist_code]), uint64(dist_bits[dist_code]), storage_ix, storage) - writeBits(uint(distnumextra), uint64(distextra), storage_ix, storage) + bw.writeBits(uint(dist_depth[dist_code]), uint64(dist_bits[dist_code])) + bw.writeBits(uint(distnumextra), uint64(distextra)) } } } -func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, bw *bitWriter) { var lit_histo histogramLiteral var cmd_histo histogramCommand var dist_histo histogramDistance @@ -1148,44 +1174,42 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, var tree []huffmanTree var num_distance_symbols uint32 = params.dist.alphabet_size - storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + storeCompressedMetaBlockHeader(is_last, length, bw) histogramClearLiteral(&lit_histo) histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) - writeBits(13, 0, storage_ix, storage) + bw.writeBits(13, 0) tree = make([]huffmanTree, maxHuffmanTreeSize) - buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], storage_ix, storage) - buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) - buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], bw) + buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], bw) + buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], bw) tree = nil - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], bw) if is_last { - jumpToByteBoundary(storage_ix, storage) + bw.jumpToByteBoundary() } } -func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, bw *bitWriter) { var num_distance_symbols uint32 = params.dist.alphabet_size var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 - storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + storeCompressedMetaBlockHeader(is_last, length, bw) - writeBits(13, 0, storage_ix, storage) + bw.writeBits(13, 0) - if n_commands <= 128 { + if len(commands) <= 128 { var histogram = [numLiteralSymbols]uint32{0} var pos uint = start_pos var num_literals uint = 0 - var i uint var lit_depth [numLiteralSymbols]byte var lit_bits [numLiteralSymbols]uint16 - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { histogram[input[pos&mask]]++ @@ -1197,11 +1221,11 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is } buildAndStoreHuffmanTreeFast(histogram[:], num_literals, /* max_bits = */ - 8, lit_depth[:], lit_bits[:], storage_ix, storage) + 8, lit_depth[:], lit_bits[:], bw) - storeStaticCommandHuffmanTree(storage_ix, storage) - storeStaticDistanceHuffmanTree(storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) + storeStaticCommandHuffmanTree(bw) + storeStaticDistanceHuffmanTree(bw) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], bw) } else { var lit_histo histogramLiteral var cmd_histo histogramCommand @@ -1215,51 +1239,45 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is histogramClearLiteral(&lit_histo) histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ - 8, lit_depth[:], lit_bits[:], storage_ix, storage) + 8, lit_depth[:], lit_bits[:], bw) buildAndStoreHuffmanTreeFast(cmd_histo.data_[:], cmd_histo.total_count_, /* max_bits = */ - 10, cmd_depth[:], cmd_bits[:], storage_ix, storage) + 10, cmd_depth[:], cmd_bits[:], bw) buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ - uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) + uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], bw) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], bw) } if is_last { - jumpToByteBoundary(storage_ix, storage) + bw.jumpToByteBoundary() } } /* This is for storing uncompressed blocks (simple raw storage of bytes-as-bytes). */ -func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, storage_ix *uint, storage []byte) { +func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, bw *bitWriter) { var masked_pos uint = position & mask - storeUncompressedMetaBlockHeader(uint(len), storage_ix, storage) - jumpToByteBoundary(storage_ix, storage) + storeUncompressedMetaBlockHeader(uint(len), bw) + bw.jumpToByteBoundary() if masked_pos+len > mask+1 { var len1 uint = mask + 1 - masked_pos - copy(storage[*storage_ix>>3:], input[masked_pos:][:len1]) - *storage_ix += len1 << 3 + bw.writeBytes(input[masked_pos:][:len1]) len -= len1 masked_pos = 0 } - copy(storage[*storage_ix>>3:], input[masked_pos:][:len]) - *storage_ix += uint(len << 3) - - /* We need to clear the next 4 bytes to continue to be - compatible with BrotliWriteBits. */ - writeBitsPrepareStorage(*storage_ix, storage) + bw.writeBytes(input[masked_pos:][:len]) /* Since the uncompressed block itself may not be the final block, add an empty one after this. */ if is_final_block { - writeBits(1, 1, storage_ix, storage) /* islast */ - writeBits(1, 1, storage_ix, storage) /* isempty */ - jumpToByteBoundary(storage_ix, storage) + bw.writeBits(1, 1) /* islast */ + bw.writeBits(1, 1) /* isempty */ + bw.jumpToByteBoundary() } } diff --git a/vendor/github.com/andybalholm/brotli/command.go b/vendor/github.com/andybalholm/brotli/command.go index e93ccdfa1e676..b1662a55552f2 100644 --- a/vendor/github.com/andybalholm/brotli/command.go +++ b/vendor/github.com/andybalholm/brotli/command.go @@ -194,26 +194,28 @@ type command struct { } /* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ -func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) { +func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) { /* Don't rely on signed int representation, use honest casts. */ var delta uint32 = uint32(byte(int8(copylen_code_delta))) - self.insert_len_ = uint32(insertlen) - self.copy_len_ = uint32(uint32(copylen) | delta<<25) + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = uint32(uint32(copylen) | delta<<25) /* The distance prefix and extra bits are stored in this Command as if npostfix and ndirect were 0, they are only recomputed later after the clustering if needed. */ - prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_) + prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) + getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_) - getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_) + return cmd } -func initInsertCommand(self *command, insertlen uint) { - self.insert_len_ = uint32(insertlen) - self.copy_len_ = 4 << 25 - self.dist_extra_ = 0 - self.dist_prefix_ = numDistanceShortCodes - getLengthCode(insertlen, 4, false, &self.cmd_prefix_) +func makeInsertCommand(insertlen uint) (cmd command) { + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = 4 << 25 + cmd.dist_extra_ = 0 + cmd.dist_prefix_ = numDistanceShortCodes + getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_) + return cmd } func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment.go b/vendor/github.com/andybalholm/brotli/compress_fragment.go index 435898e1651d5..dbf0c43bf2692 100644 --- a/vendor/github.com/andybalholm/brotli/compress_fragment.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment.go @@ -33,14 +33,8 @@ func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 { } func isMatch5(p1 []byte, p2 []byte) bool { - var i int - for i = 0; i < 5; i++ { - if p1[i] != p2[i] { - return false - } - } - - return true + return binary.LittleEndian.Uint32(p1) == binary.LittleEndian.Uint32(p2) && + p1[4] == p2[4] } /* Builds a literal prefix code into "depths" and "bits" based on the statistics @@ -51,7 +45,7 @@ func isMatch5(p1 []byte, p2 []byte) bool { and thus have to assign a non-zero depth for each literal. Returns estimated compression ratio millibytes/char for encoding given input with generated code. */ -func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint { +func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, bw *bitWriter) uint { var histogram = [256]uint32{0} var histogram_total uint var i uint @@ -88,7 +82,7 @@ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte } buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */ - 8, depths, bits, storage_ix, storage) + 8, depths, bits, bw) { var literal_ratio uint = 0 for i = 0; i < 256; i++ { @@ -104,7 +98,7 @@ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte /* Builds a command and distance prefix code (each 64 symbols) into "depth" and "bits" based on "histogram" and stores it into the bit stream. */ -func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { +func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, bw *bitWriter) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ @@ -151,141 +145,141 @@ func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []ui cmd_depth[448+8*i] = depth[56+i] } - storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], bw) } - storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) + storeHuffmanTree(depth[64:], 64, tree[:], bw) } /* REQUIRES: insertlen < 6210 */ -func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { +func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, bw *bitWriter) { if insertlen < 6 { var code uint = insertlen + 40 - writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + bw.writeBits(uint(depth[code]), uint64(bits[code])) histo[code]++ } else if insertlen < 130 { var tail uint = insertlen - 2 var nbits uint32 = log2FloorNonZero(tail) - 1 var prefix uint = tail >> nbits var inscode uint = uint((nbits << 1) + uint32(prefix) + 42) - writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage) - writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits var code uint = uint((nbits << 1) + uint32(prefix) + 20) - writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) - writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits var code uint = uint((nbits << 1) + uint32(prefix) + 4) - writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) - writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> 5) + 30 - writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) - writeBits(5, uint64(tail)&31, storage_ix, storage) - writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) + bw.writeBits(uint(depth[code]), uint64(bits[code])) + bw.writeBits(5, uint64(tail)&31) + bw.writeBits(uint(depth[64]), uint64(bits[64])) histo[code]++ histo[64]++ } else if copylen < 2120 { var tail uint = copylen - 72 var nbits uint32 = log2FloorNonZero(tail) var code uint = uint(nbits + 28) - writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) - writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<> nbits) & 1 var offset uint = (2 + prefix) << nbits var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80) - writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage) - writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage) + bw.writeBits(uint(depth[distcode]), uint64(bits[distcode])) + bw.writeBits(uint(nbits), uint64(d)-uint64(offset)) histo[distcode]++ } -func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { +func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, bw *bitWriter) { var j uint for j = 0; j < len; j++ { var lit byte = input[j] - writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage) + bw.writeBits(uint(depth[lit]), uint64(bits[lit])) } } /* REQUIRES: len <= 1 << 24. */ -func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { +func storeMetaBlockHeader1(len uint, is_uncompressed bool, bw *bitWriter) { var nibbles uint = 6 /* ISLAST */ - writeBits(1, 0, storage_ix, storage) + bw.writeBits(1, 0) if len <= 1<<16 { nibbles = 4 @@ -293,34 +287,11 @@ func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, sto nibbles = 5 } - writeBits(2, uint64(nibbles)-4, storage_ix, storage) - writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) + bw.writeBits(2, uint64(nibbles)-4) + bw.writeBits(nibbles*4, uint64(len)-1) /* ISUNCOMPRESSED */ - writeSingleBit(is_uncompressed, storage_ix, storage) -} - -func updateBits(n_bits uint, bits uint32, pos uint, array []byte) { - for n_bits > 0 { - var byte_pos uint = pos >> 3 - var n_unchanged_bits uint = pos & 7 - var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) - var total_bits uint = n_unchanged_bits + n_changed_bits - var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) - var unchanged_bits uint32 = uint32(array[byte_pos]) & mask - var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) - array[byte_pos] = byte(changed_bits<>= n_changed_bits - pos += n_changed_bits - } -} - -func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) { - var bitpos uint = new_storage_ix & 7 - var mask uint = (1 << bitpos) - 1 - storage[new_storage_ix>>3] &= byte(mask) - *storage_ix = new_storage_ix + bw.writeSingleBit(is_uncompressed) } var shouldMergeBlock_kSampleRate uint = 43 @@ -351,151 +322,26 @@ func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertl } } -func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) { - var len uint = uint(-cap(end) + cap(begin)) - rewindBitPosition1(storage_ix_start, storage_ix, storage) - storeMetaBlockHeader1(uint(len), true, storage_ix, storage) - *storage_ix = (*storage_ix + 7) &^ 7 - copy(storage[*storage_ix>>3:], begin[:len]) - *storage_ix += uint(len << 3) - storage[*storage_ix>>3] = 0 +func emitUncompressedMetaBlock1(data []byte, storage_ix_start uint, bw *bitWriter) { + bw.rewind(storage_ix_start) + storeMetaBlockHeader1(uint(len(data)), true, bw) + bw.jumpToByteBoundary() + bw.writeBytes(data) } var kCmdHistoSeed = [128]uint32{ - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 0, } var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15 var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16 -func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { +func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, bw *bitWriter) { var cmd_histo [128]uint32 var ip_end int var next_emit int = 0 @@ -506,7 +352,7 @@ func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table [] var metablock_start int = input var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) var total_block_size uint = block_size - var mlen_storage_ix uint = *storage_ix + 3 + var mlen_storage_ix uint = bw.getPos() + 3 var lit_depth [256]byte var lit_bits [256]uint16 var literal_ratio uint @@ -523,21 +369,21 @@ func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table [] /* Save the bit position of the MLEN field of the meta-block header, so that we can update it later if we decide to extend this meta-block. */ - storeMetaBlockHeader1(block_size, false, storage_ix, storage) + storeMetaBlockHeader1(block_size, false, bw) /* No block splits, no contexts. */ - writeBits(13, 0, storage_ix, storage) + bw.writeBits(13, 0) - literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], bw) { /* Store the pre-compressed command and distance prefix codes. */ var i uint for i = 0; i+7 < *cmd_code_numbits; i += 8 { - writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage) + bw.writeBits(8, uint64(cmd_code[i>>3])) } } - writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage) + bw.writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3])) /* Initialize the command and distance histograms. We will gather statistics of command and distance codes during the processing @@ -636,27 +482,27 @@ emit_commands: var insert uint = uint(base - next_emit) ip += int(matched) if insert < 6210 { - emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { - emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage) + emitUncompressedMetaBlock1(in[metablock_start:base], mlen_storage_ix-3, bw) input_size -= uint(base - input) input = base next_emit = input goto next_block } else { - emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) } - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) if distance == last_distance { - writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage) + bw.writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64])) cmd_histo[64]++ } else { - emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], bw) last_distance = distance } - emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], bw) next_emit = ip if ip >= ip_limit { @@ -692,8 +538,8 @@ emit_commands: } ip += int(matched) last_distance = int(base - candidate) /* > 0 */ - emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) - emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], bw) next_emit = ip if ip >= ip_limit { @@ -739,7 +585,7 @@ emit_remainder: nibbles. */ total_block_size += block_size - updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage) + bw.updateBits(20, uint32(total_block_size-1), mlen_storage_ix) goto emit_commands } @@ -747,13 +593,13 @@ emit_remainder: if next_emit < ip_end { var insert uint = uint(ip_end - next_emit) if insert < 6210 { - emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { - emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage) + emitUncompressedMetaBlock1(in[metablock_start:ip_end], mlen_storage_ix-3, bw) } else { - emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) } } @@ -769,30 +615,29 @@ next_block: /* Save the bit position of the MLEN field of the meta-block header, so that we can update it later if we decide to extend this meta-block. */ - mlen_storage_ix = *storage_ix + 3 + mlen_storage_ix = bw.getPos() + 3 - storeMetaBlockHeader1(block_size, false, storage_ix, storage) + storeMetaBlockHeader1(block_size, false, bw) /* No block splits, no contexts. */ - writeBits(13, 0, storage_ix, storage) + bw.writeBits(13, 0) - literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) - buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage) + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], bw) + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, bw) goto emit_commands } if !is_last { /* If this is not the last block, update the command and distance prefix codes for the next block and store the compressed forms. */ - cmd_code[0] = 0 - - *cmd_code_numbits = 0 - buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code) + var bw bitWriter + bw.dst = cmd_code + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, &bw) + *cmd_code_numbits = bw.getPos() } } -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. +/* Compresses "input" string to bw as one or more complete meta-blocks. If "is_last" is 1, emits an additional empty last meta-block. @@ -813,28 +658,28 @@ next_block: REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two OUTPUT: maximal copy distance <= |input_size| OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ -func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { - var initial_storage_ix uint = *storage_ix +func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, bw *bitWriter) { + var initial_storage_ix uint = bw.getPos() var table_bits uint = uint(log2FloorNonZero(table_size)) if input_size == 0 { assert(is_last) - writeBits(1, 1, storage_ix, storage) /* islast */ - writeBits(1, 1, storage_ix, storage) /* isempty */ - *storage_ix = (*storage_ix + 7) &^ 7 + bw.writeBits(1, 1) /* islast */ + bw.writeBits(1, 1) /* isempty */ + bw.jumpToByteBoundary() return } - compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage) + compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, bw) /* If output is larger than single uncompressed block, rewrite it. */ - if *storage_ix-initial_storage_ix > 31+(input_size<<3) { - emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage) + if bw.getPos()-initial_storage_ix > 31+(input_size<<3) { + emitUncompressedMetaBlock1(input[:input_size], initial_storage_ix, bw) } if is_last { - writeBits(1, 1, storage_ix, storage) /* islast */ - writeBits(1, 1, storage_ix, storage) /* isempty */ - *storage_ix = (*storage_ix + 7) &^ 7 + bw.writeBits(1, 1) /* islast */ + bw.writeBits(1, 1) /* isempty */ + bw.jumpToByteBoundary() } } diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go index ffeb321644bc5..2473aca3fec8f 100644 --- a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go @@ -30,19 +30,18 @@ func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { } func isMatch1(p1 []byte, p2 []byte, length uint) bool { - var i uint - for i = 0; i < length && i < 6; i++ { - if p1[i] != p2[i] { - return false - } + if binary.LittleEndian.Uint32(p1) != binary.LittleEndian.Uint32(p2) { + return false } - - return true + if length == 4 { + return true + } + return p1[4] == p2[4] && p1[5] == p2[5] } /* Builds a command and distance prefix code (each 64 symbols) into "depth" and "bits" based on "histogram" and stores it into the bit stream. */ -func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { +func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, bw *bitWriter) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ @@ -88,10 +87,10 @@ func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uin cmd_depth[448+8*i] = depth[16+i] } - storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], bw) } - storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) + storeHuffmanTree(depth[64:], 64, tree[:], bw) } func emitInsertLen(insertlen uint32, commands *[]uint32) { @@ -198,11 +197,11 @@ func emitDistance(distance uint32, commands *[]uint32) { } /* REQUIRES: len <= 1 << 24. */ -func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { +func storeMetaBlockHeader(len uint, is_uncompressed bool, bw *bitWriter) { var nibbles uint = 6 /* ISLAST */ - writeBits(1, 0, storage_ix, storage) + bw.writeBits(1, 0) if len <= 1<<16 { nibbles = 4 @@ -210,11 +209,11 @@ func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, stor nibbles = 5 } - writeBits(2, uint64(nibbles)-4, storage_ix, storage) - writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) + bw.writeBits(2, uint64(nibbles)-4) + bw.writeBits(nibbles*4, uint64(len)-1) /* ISUNCOMPRESSED */ - writeSingleBit(is_uncompressed, storage_ix, storage) + bw.writeSingleBit(is_uncompressed) } func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) { @@ -441,163 +440,20 @@ emit_remainder: } var storeCommands_kNumExtraBits = [128]uint32{ - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 2, - 2, - 3, - 3, - 4, - 4, - 5, - 5, - 6, - 7, - 8, - 9, - 10, - 12, - 14, - 24, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 2, - 2, - 3, - 3, - 4, - 4, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 2, - 2, - 3, - 3, - 4, - 4, - 5, - 5, - 6, - 7, - 8, - 9, - 10, - 24, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 2, - 2, - 3, - 3, - 4, - 4, - 5, - 5, - 6, - 6, - 7, - 7, - 8, - 8, - 9, - 9, - 10, - 10, - 11, - 11, - 12, - 12, - 13, - 13, - 14, - 14, - 15, - 15, - 16, - 16, - 17, - 17, - 18, - 18, - 19, - 19, - 20, - 20, - 21, - 21, - 22, - 22, - 23, - 23, - 24, - 24, + 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, + 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, } var storeCommands_kInsertOffset = [24]uint32{ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 8, - 10, - 14, - 18, - 26, - 34, - 50, - 66, - 98, - 130, - 194, - 322, - 578, - 1090, - 2114, - 6210, - 22594, + 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, + 1090, 2114, 6210, 22594, } -func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) { +func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, bw *bitWriter) { var lit_depths [256]byte var lit_bits [256]uint16 var lit_histo = [256]uint32{0} @@ -610,7 +466,7 @@ func storeCommands(literals []byte, num_literals uint, commands []uint32, num_co } buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */ - 8, lit_depths[:], lit_bits[:], storage_ix, storage) + 8, lit_depths[:], lit_bits[:], bw) for i = 0; i < num_commands; i++ { var code uint32 = commands[i] & 0xFF @@ -622,21 +478,21 @@ func storeCommands(literals []byte, num_literals uint, commands []uint32, num_co cmd_histo[2] += 1 cmd_histo[64] += 1 cmd_histo[84] += 1 - buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage) + buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], bw) for i = 0; i < num_commands; i++ { var cmd uint32 = commands[i] var code uint32 = cmd & 0xFF var extra uint32 = cmd >> 8 assert(code < 128) - writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage) - writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage) + bw.writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code])) + bw.writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra)) if code < 24 { var insert uint32 = storeCommands_kInsertOffset[code] + extra var j uint32 for j = 0; j < insert; j++ { var lit byte = literals[0] - writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage) + bw.writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit])) literals = literals[1:] } } @@ -664,22 +520,13 @@ func shouldCompress(input []byte, input_size uint, num_literals uint) bool { } } -func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) { - var bitpos uint = new_storage_ix & 7 - var mask uint = (1 << bitpos) - 1 - storage[new_storage_ix>>3] &= byte(mask) - *storage_ix = new_storage_ix -} - -func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) { - storeMetaBlockHeader(input_size, true, storage_ix, storage) - *storage_ix = (*storage_ix + 7) &^ 7 - copy(storage[*storage_ix>>3:], input[:input_size]) - *storage_ix += input_size << 3 - storage[*storage_ix>>3] = 0 +func emitUncompressedMetaBlock(input []byte, input_size uint, bw *bitWriter) { + storeMetaBlockHeader(input_size, true, bw) + bw.jumpToByteBoundary() + bw.writeBytes(input[:input_size]) } -func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) { +func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, bw *bitWriter) { /* Save the start of the first block for position and distance computations. */ var base_ip []byte = input @@ -693,17 +540,17 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co num_literals = uint(-cap(literals) + cap(literal_buf)) if shouldCompress(input, block_size, num_literals) { var num_commands uint = uint(-cap(commands) + cap(command_buf)) - storeMetaBlockHeader(block_size, false, storage_ix, storage) + storeMetaBlockHeader(block_size, false, bw) /* No block splits, no contexts. */ - writeBits(13, 0, storage_ix, storage) + bw.writeBits(13, 0) - storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage) + storeCommands(literal_buf, num_literals, command_buf, num_commands, bw) } else { /* Since we did not find many backward references and the entropy of the data is close to 8 bits, we can simply emit an uncompressed block. This makes compression speed of uncompressible data about 3x faster. */ - emitUncompressedMetaBlock(input, block_size, storage_ix, storage) + emitUncompressedMetaBlock(input, block_size, bw) } input = input[block_size:] @@ -711,8 +558,7 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co } } -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. +/* Compresses "input" string to bw as one or more complete meta-blocks. If "is_last" is 1, emits an additional empty last meta-block. @@ -724,8 +570,8 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co REQUIRES: "table_size" is a power of two OUTPUT: maximal copy distance <= |input_size| OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ -func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { - var initial_storage_ix uint = *storage_ix +func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, bw *bitWriter) { + var initial_storage_ix uint = bw.getPos() var table_bits uint = uint(log2FloorNonZero(table_size)) var min_match uint if table_bits <= 15 { @@ -733,17 +579,17 @@ func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, comman } else { min_match = 6 } - compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage) + compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, bw) /* If output is larger than single uncompressed block, rewrite it. */ - if *storage_ix-initial_storage_ix > 31+(input_size<<3) { - rewindBitPosition(initial_storage_ix, storage_ix, storage) - emitUncompressedMetaBlock(input, input_size, storage_ix, storage) + if bw.getPos()-initial_storage_ix > 31+(input_size<<3) { + bw.rewind(initial_storage_ix) + emitUncompressedMetaBlock(input, input_size, bw) } if is_last { - writeBits(1, 1, storage_ix, storage) /* islast */ - writeBits(1, 1, storage_ix, storage) /* isempty */ - *storage_ix = (*storage_ix + 7) &^ 7 + bw.writeBits(1, 1) /* islast */ + bw.writeBits(1, 1) /* isempty */ + bw.jumpToByteBoundary() } } diff --git a/vendor/github.com/andybalholm/brotli/encode.go b/vendor/github.com/andybalholm/brotli/encode.go index c01322bf14768..3abaf571aff0a 100644 --- a/vendor/github.com/andybalholm/brotli/encode.go +++ b/vendor/github.com/andybalholm/brotli/encode.go @@ -74,26 +74,22 @@ const ( type Writer struct { dst io.Writer options WriterOptions + err error params encoderParams hasher_ hasherHandle input_pos_ uint64 ringbuffer_ ringBuffer - cmd_alloc_size_ uint - commands_ []command - num_commands_ uint + commands []command num_literals_ uint last_insert_len_ uint last_flush_pos_ uint64 last_processed_pos_ uint64 dist_cache_ [numDistanceShortCodes]int saved_dist_cache_ [4]int - last_bytes_ uint16 - last_bytes_bits_ byte prev_byte_ byte prev_byte2_ byte - storage_size_ uint - storage_ []byte + bw bitWriter small_table_ [1 << 10]int large_table_ []int large_table_size_ uint @@ -103,9 +99,6 @@ type Writer struct { cmd_code_numbits_ uint command_buf_ []uint32 literal_buf_ []byte - next_out_ []byte - available_out_ uint - total_out_ uint tiny_buf_ struct { u64 [2]uint64 u8 [16]byte @@ -146,16 +139,6 @@ func wrapPosition(position uint64) uint32 { return result } -func getBrotliStorage(s *Writer, size uint) []byte { - if s.storage_size_ < size { - s.storage_ = nil - s.storage_ = make([]byte, size) - s.storage_size_ = size - } - - return s.storage_ -} - func hashTableSize(max_table_size uint, input_size uint) uint { var htsize uint = 256 for htsize < max_table_size && htsize < input_size { @@ -201,353 +184,22 @@ func getHashTable(s *Writer, quality int, input_size uint, table_size *uint) []i return table } -func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) { +func encodeWindowBits(lgwin int, large_window bool, bw *bitWriter) { if large_window { - *last_bytes = uint16((lgwin&0x3F)<<8 | 0x11) - *last_bytes_bits = 14 + bw.writeBits(14, uint64((lgwin&0x3F)<<8|0x11)) } else { if lgwin == 16 { - *last_bytes = 0 - *last_bytes_bits = 1 + bw.writeBits(1, 0) } else if lgwin == 17 { - *last_bytes = 1 - *last_bytes_bits = 7 + bw.writeBits(7, 1) } else if lgwin > 17 { - *last_bytes = uint16((lgwin-17)<<1 | 0x01) - *last_bytes_bits = 4 + bw.writeBits(4, uint64((lgwin-17)<<1|0x01)) } else { - *last_bytes = uint16((lgwin-8)<<4 | 0x01) - *last_bytes_bits = 7 + bw.writeBits(7, uint64((lgwin-8)<<4|0x01)) } } } -/* Initializes the command and distance prefix codes for the first block. */ - -var initCommandPrefixCodes_kDefaultCommandDepths = [128]byte{ - 0, - 4, - 4, - 5, - 6, - 6, - 7, - 7, - 7, - 7, - 7, - 8, - 8, - 8, - 8, - 8, - 0, - 0, - 0, - 4, - 4, - 4, - 4, - 4, - 5, - 5, - 6, - 6, - 6, - 6, - 7, - 7, - 7, - 7, - 10, - 10, - 10, - 10, - 10, - 10, - 0, - 4, - 4, - 5, - 5, - 5, - 6, - 6, - 7, - 8, - 8, - 9, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 10, - 5, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 6, - 6, - 6, - 6, - 6, - 6, - 5, - 5, - 5, - 5, - 5, - 5, - 4, - 4, - 4, - 4, - 4, - 4, - 4, - 5, - 5, - 5, - 5, - 5, - 5, - 6, - 6, - 7, - 7, - 7, - 8, - 10, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, - 12, -} -var initCommandPrefixCodes_kDefaultCommandBits = [128]uint16{ - 0, - 0, - 8, - 9, - 3, - 35, - 7, - 71, - 39, - 103, - 23, - 47, - 175, - 111, - 239, - 31, - 0, - 0, - 0, - 4, - 12, - 2, - 10, - 6, - 13, - 29, - 11, - 43, - 27, - 59, - 87, - 55, - 15, - 79, - 319, - 831, - 191, - 703, - 447, - 959, - 0, - 14, - 1, - 25, - 5, - 21, - 19, - 51, - 119, - 159, - 95, - 223, - 479, - 991, - 63, - 575, - 127, - 639, - 383, - 895, - 255, - 767, - 511, - 1023, - 14, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 27, - 59, - 7, - 39, - 23, - 55, - 30, - 1, - 17, - 9, - 25, - 5, - 0, - 8, - 4, - 12, - 2, - 10, - 6, - 21, - 13, - 29, - 3, - 19, - 11, - 15, - 47, - 31, - 95, - 63, - 127, - 255, - 767, - 2815, - 1791, - 3839, - 511, - 2559, - 1535, - 3583, - 1023, - 3071, - 2047, - 4095, -} -var initCommandPrefixCodes_kDefaultCommandCode = []byte{ - 0xff, - 0x77, - 0xd5, - 0xbf, - 0xe7, - 0xde, - 0xea, - 0x9e, - 0x51, - 0x5d, - 0xde, - 0xc6, - 0x70, - 0x57, - 0xbc, - 0x58, - 0x58, - 0x58, - 0xd8, - 0xd8, - 0x58, - 0xd5, - 0xcb, - 0x8c, - 0xea, - 0xe0, - 0xc3, - 0x87, - 0x1f, - 0x83, - 0xc1, - 0x60, - 0x1c, - 0x67, - 0xb2, - 0xaa, - 0x06, - 0x83, - 0xc1, - 0x60, - 0x30, - 0x18, - 0xcc, - 0xa1, - 0xce, - 0x88, - 0x54, - 0x94, - 0x46, - 0xe1, - 0xb0, - 0xd0, - 0x4e, - 0xb2, - 0xf7, - 0x04, - 0x00, -} -var initCommandPrefixCodes_kDefaultCommandCodeNumBits uint = 448 - -func initCommandPrefixCodes(cmd_depths []byte, cmd_bits []uint16, cmd_code []byte, cmd_code_numbits *uint) { - copy(cmd_depths, initCommandPrefixCodes_kDefaultCommandDepths[:]) - copy(cmd_bits, initCommandPrefixCodes_kDefaultCommandBits[:]) - - /* Initialize the pre-compressed form of the command and distance prefix - codes. */ - copy(cmd_code, initCommandPrefixCodes_kDefaultCommandCode) - - *cmd_code_numbits = initCommandPrefixCodes_kDefaultCommandCodeNumBits -} - /* Decide about the context map based on the ability of the prediction ability of the previous byte UTF8-prefix on the next byte. The prediction ability is calculated as Shannon entropy. Here we need @@ -557,136 +209,16 @@ func initCommandPrefixCodes(cmd_depths []byte, cmd_bits []uint16, cmd_code []byt coding. */ var kStaticContextMapContinuation = [64]uint32{ - 1, - 1, - 2, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, + 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } var kStaticContextMapSimpleUTF8 = [64]uint32{ - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, + 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) { @@ -738,70 +270,22 @@ func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts * first 5 bits of literals. */ var kStaticContextMapComplexUTF8 = [64]uint32{ - 11, - 11, - 12, - 12, - 0, - 0, - 0, - 0, - 1, - 1, - 9, - 9, - 2, - 2, - 2, - 2, - 1, - 1, - 1, - 1, - 8, - 3, - 3, - 3, - 1, - 1, - 1, - 1, - 2, - 2, - 2, - 2, - 8, - 4, - 4, - 4, - 8, - 7, - 4, - 4, - 8, - 0, - 0, - 0, - 3, - 3, - 3, - 3, - 5, - 5, - 10, - 5, - 5, - 5, - 10, - 5, - 6, - 6, - 6, - 6, - 6, - 6, - 6, - 6, + 11, 11, 12, 12, /* 0 special */ + 0, 0, 0, 0, /* 4 lf */ + 1, 1, 9, 9, /* 8 space */ + 2, 2, 2, 2, /* !, first after space/lf and after something else. */ + 1, 1, 1, 1, /* " */ + 8, 3, 3, 3, /* % */ + 1, 1, 1, 1, /* ({[ */ + 2, 2, 2, 2, /* }]) */ + 8, 4, 4, 4, /* :; */ + 8, 7, 4, 4, /* . */ + 8, 0, 0, 0, /* > */ + 3, 3, 3, 3, /* [0..9] */ + 5, 5, 10, 5, /* [A-Z] */ + 5, 5, 10, 5, + 6, 6, 6, 6, /* [a-z] */ + 6, 6, 6, 6, } func shouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool { @@ -933,40 +417,34 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, return contextUTF8 } -func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { +func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, bw *bitWriter) { var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) - var last_bytes uint16 - var last_bytes_bits byte var literal_context_lut contextLUT = getContextLUT(literal_context_mode) var block_params encoderParams = *params if bytes == 0 { /* Write the ISLAST and ISEMPTY bits. */ - writeBits(2, 3, storage_ix, storage) - - *storage_ix = (*storage_ix + 7) &^ 7 + bw.writeBits(2, 3) + bw.jumpToByteBoundary() return } - if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) { + if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) { /* Restore the distance cache, as its last update by CreateBackwardReferences is now unused. */ copy(dist_cache, saved_dist_cache[:4]) - storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, bw) return } - assert(*storage_ix <= 14) - last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) - last_bytes_bits = byte(*storage_ix) + savedPos := bw.getPos() if params.quality <= maxQualityForStaticEntropyCodes { - storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, bw) } else if params.quality < minQualityForBlockSplit { - storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, bw) } else { - var mb metaBlockSplit - initMetaBlockSplit(&mb) + mb := getMetaBlockSplit() if params.quality < minQualityForHqBlockSplitting { var num_literal_contexts uint = 1 var literal_context_map []uint32 = nil @@ -974,9 +452,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map) } - buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb) + buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, mb) } else { - buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb) + buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, mb) } if params.quality >= minQualityForOptimizeHistograms { @@ -988,21 +466,18 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes num_effective_dist_codes = numHistogramDistanceSymbols } - optimizeHistograms(num_effective_dist_codes, &mb) + optimizeHistograms(num_effective_dist_codes, mb) } - storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage) - destroyMetaBlockSplit(&mb) + storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, bw) + freeMetaBlockSplit(mb) } - if bytes+4 < *storage_ix>>3 { + if bytes+4 < bw.getPos()>>3 { /* Restore the distance cache and last byte. */ copy(dist_cache, saved_dist_cache[:4]) - - storage[0] = byte(last_bytes) - storage[1] = byte(last_bytes >> 8) - *storage_ix = uint(last_bytes_bits) - storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) + bw.rewind(savedPos) + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, bw) } } @@ -1035,8 +510,10 @@ func ensureInitialized(s *Writer) bool { return true } - s.last_bytes_bits_ = 0 - s.last_bytes_ = 0 + s.bw.bits = 0 + s.bw.nbits = 0 + s.bw.dst = s.bw.dst[:0] + s.remaining_metadata_bytes_ = math.MaxUint32 sanitizeParams(&s.params) @@ -1052,11 +529,42 @@ func ensureInitialized(s *Writer) bool { lgwin = brotli_max_int(lgwin, 18) } - encodeWindowBits(lgwin, s.params.large_window, &s.last_bytes_, &s.last_bytes_bits_) + encodeWindowBits(lgwin, s.params.large_window, &s.bw) } if s.params.quality == fastOnePassCompressionQuality { - initCommandPrefixCodes(s.cmd_depths_[:], s.cmd_bits_[:], s.cmd_code_[:], &s.cmd_code_numbits_) + s.cmd_depths_ = [128]byte{ + 0, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, + 0, 0, 0, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, + 7, 7, 10, 10, 10, 10, 10, 10, 0, 4, 4, 5, 5, 5, 6, 6, + 7, 8, 8, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, + 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 10, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + } + s.cmd_bits_ = [128]uint16{ + 0, 0, 8, 9, 3, 35, 7, 71, + 39, 103, 23, 47, 175, 111, 239, 31, + 0, 0, 0, 4, 12, 2, 10, 6, + 13, 29, 11, 43, 27, 59, 87, 55, + 15, 79, 319, 831, 191, 703, 447, 959, + 0, 14, 1, 25, 5, 21, 19, 51, + 119, 159, 95, 223, 479, 991, 63, 575, + 127, 639, 383, 895, 255, 767, 511, 1023, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 27, 59, 7, 39, 23, 55, 30, 1, 17, 9, 25, 5, 0, 8, 4, 12, + 2, 10, 6, 21, 13, 29, 3, 19, 11, 15, 47, 31, 95, 63, 127, 255, + 767, 2815, 1791, 3839, 511, 2559, 1535, 3583, 1023, 3071, 2047, 4095, + } + s.cmd_code_ = [512]byte{ + 0xff, 0x77, 0xd5, 0xbf, 0xe7, 0xde, 0xea, 0x9e, 0x51, 0x5d, 0xde, 0xc6, + 0x70, 0x57, 0xbc, 0x58, 0x58, 0x58, 0xd8, 0xd8, 0x58, 0xd5, 0xcb, 0x8c, + 0xea, 0xe0, 0xc3, 0x87, 0x1f, 0x83, 0xc1, 0x60, 0x1c, 0x67, 0xb2, 0xaa, + 0x06, 0x83, 0xc1, 0x60, 0x30, 0x18, 0xcc, 0xa1, 0xce, 0x88, 0x54, 0x94, + 0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x04, 0x00, + } + s.cmd_code_numbits_ = 448 } s.is_initialized_ = true @@ -1081,33 +589,23 @@ func encoderInitParams(params *encoderParams) { func encoderInitState(s *Writer) { encoderInitParams(&s.params) s.input_pos_ = 0 - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 s.last_insert_len_ = 0 s.last_flush_pos_ = 0 s.last_processed_pos_ = 0 s.prev_byte_ = 0 s.prev_byte2_ = 0 - s.storage_size_ = 0 - s.storage_ = nil - s.hasher_ = nil - s.large_table_ = nil - s.large_table_size_ = 0 + if s.hasher_ != nil { + s.hasher_.Common().is_prepared_ = false + } s.cmd_code_numbits_ = 0 - s.command_buf_ = nil - s.literal_buf_ = nil - s.next_out_ = nil - s.available_out_ = 0 - s.total_out_ = 0 s.stream_state_ = streamProcessing s.is_last_block_emitted_ = false s.is_initialized_ = false ringBufferInit(&s.ringbuffer_) - s.commands_ = nil - s.cmd_alloc_size_ = 0 - /* Initialize distance cache. */ s.dist_cache_[0] = 4 @@ -1190,7 +688,7 @@ func updateLastProcessedPos(s *Writer) bool { } func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) { - var last_command *command = &s.commands_[s.num_commands_-1] + var last_command *command = &s.commands[len(s.commands)-1] var data []byte = s.ringbuffer_.buffer_ var mask uint32 = s.ringbuffer_.mask_ var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap @@ -1219,18 +717,17 @@ func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uin } /* - Processes the accumulated input data and sets |*out_size| to the length of - the new output meta-block, or to zero if no new output meta-block has been - created (in this case the processed input data is buffered internally). - If |*out_size| is positive, |*output| points to the start of the output - data. If |is_last| or |force_flush| is true, an output meta-block is + Processes the accumulated input data and writes + the new output meta-block to s.dest, if one has been + created (otherwise the processed input data is buffered internally). + If |is_last| or |force_flush| is true, an output meta-block is always created. However, until |is_last| is true encoder may retain up to 7 bits of the last byte of output. To force encoder to dump the remaining bits use WriteMetadata() to append an empty meta-data block. Returns false if the size of the input data is larger than input_block_size(). */ -func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, output *[]byte) bool { +func encodeData(s *Writer, is_last bool, force_flush bool) bool { var delta uint64 = unprocessedInputSize(s) var bytes uint32 = uint32(delta) var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_) @@ -1253,60 +750,52 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu return false } - if s.params.quality == fastTwoPassCompressionQuality && s.command_buf_ == nil { - s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) - s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) + if s.params.quality == fastTwoPassCompressionQuality { + if s.command_buf_ == nil || cap(s.command_buf_) < int(kCompressFragmentTwoPassBlockSize) { + s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) + s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) + } else { + s.command_buf_ = s.command_buf_[:kCompressFragmentTwoPassBlockSize] + s.literal_buf_ = s.literal_buf_[:kCompressFragmentTwoPassBlockSize] + } } if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality { - var storage []byte - var storage_ix uint = uint(s.last_bytes_bits_) var table_size uint var table []int if delta == 0 && !is_last { /* We have no new input data and we don't have to finish the stream, so nothing to do. */ - *out_size = 0 - return true } - storage = getBrotliStorage(s, uint(2*bytes+503)) - storage[0] = byte(s.last_bytes_) - storage[1] = byte(s.last_bytes_ >> 8) table = getHashTable(s, s.params.quality, uint(bytes), &table_size) if s.params.quality == fastOnePassCompressionQuality { - compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) + compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &s.bw) } else { - compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage) + compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &s.bw) } - s.last_bytes_ = uint16(storage[storage_ix>>3]) - s.last_bytes_bits_ = byte(storage_ix & 7) updateLastProcessedPos(s) - *output = storage[0:] - *out_size = storage_ix >> 3 + s.writeOutput(s.bw.dst) + s.bw.dst = s.bw.dst[:0] return true } { /* Theoretical max number of commands is 1 per 2 bytes. */ - var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1) - if newsize > s.cmd_alloc_size_ { - var new_commands []command - + newsize := len(s.commands) + int(bytes)/2 + 1 + if newsize > cap(s.commands) { /* Reserve a bit more memory to allow merging with a next block without reallocation: that would impact speed. */ - newsize += uint((bytes / 4) + 16) + newsize += int(bytes/4) + 16 - s.cmd_alloc_size_ = newsize - new_commands = make([]command, newsize) - if s.commands_ != nil { - copy(new_commands, s.commands_[:s.num_commands_]) - s.commands_ = nil + new_commands := make([]command, len(s.commands), newsize) + if s.commands != nil { + copy(new_commands, s.commands) } - s.commands_ = new_commands + s.commands = new_commands } } @@ -1314,46 +803,44 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_)) - if s.num_commands_ != 0 && s.last_insert_len_ == 0 { + if len(s.commands) != 0 && s.last_insert_len_ == 0 { extendLastCommand(s, &bytes, &wrapped_last_processed_pos) } if s.params.quality == zopflificationQuality { assert(s.params.hasher.type_ == 10) - createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else if s.params.quality == hqZopflificationQuality { assert(s.params.hasher.type_ == 10) - createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else { - createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } { var max_length uint = maxMetablockSize(&s.params) var max_literals uint = max_length / 8 - var max_commands uint = max_length / 8 + max_commands := int(max_length / 8) var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_) var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length) - var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols) + var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols) /* If maximal possible additional block doesn't fit metablock, flush now. */ /* TODO: Postpone decision until next block arrives? */ /* If block splitting is not used, then flush as soon as there is some amount of commands / literals produced. */ - if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands { + if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands { /* Merge with next input block. Everything will happen later. */ if updateLastProcessedPos(s) { hasherReset(s.hasher_) } - *out_size = 0 return true } } /* Create the last insert-only command. */ if s.last_insert_len_ > 0 { - initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_) - s.num_commands_++ + s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_)) s.num_literals_ += s.last_insert_len_ s.last_insert_len_ = 0 } @@ -1361,8 +848,6 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu if !is_last && s.input_pos_ == s.last_flush_pos_ { /* We have no new input data and we don't have to finish the stream, so nothing to do. */ - *out_size = 0 - return true } @@ -1371,13 +856,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu assert(s.input_pos_-s.last_flush_pos_ <= 1<<24) { var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_) - var storage []byte = getBrotliStorage(s, uint(2*metablock_size+503)) - var storage_ix uint = uint(s.last_bytes_bits_) - storage[0] = byte(s.last_bytes_) - storage[1] = byte(s.last_bytes_ >> 8) - writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) - s.last_bytes_ = uint16(storage[storage_ix>>3]) - s.last_bytes_bits_ = byte(storage_ix & 7) + writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &s.bw) s.last_flush_pos_ = s.input_pos_ if updateLastProcessedPos(s) { hasherReset(s.hasher_) @@ -1391,36 +870,29 @@ func encodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, outpu s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask] } - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 /* Save the state of the distance cache in case we need to restore it for emitting an uncompressed block. */ copy(s.saved_dist_cache_[:], s.dist_cache_[:]) - *output = storage[0:] - *out_size = storage_ix >> 3 + s.writeOutput(s.bw.dst) + s.bw.dst = s.bw.dst[:0] return true } } -/* Dumps remaining output bits and metadata header to |header|. - Returns number of produced bytes. - REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. +/* Dumps remaining output bits and metadata header to s.bw. REQUIRED: |block_size| <= (1 << 24). */ -func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { - var storage_ix uint - storage_ix = uint(s.last_bytes_bits_) - header[0] = byte(s.last_bytes_) - header[1] = byte(s.last_bytes_ >> 8) - s.last_bytes_ = 0 - s.last_bytes_bits_ = 0 - - writeBits(1, 0, &storage_ix, header) - writeBits(2, 3, &storage_ix, header) - writeBits(1, 0, &storage_ix, header) +func writeMetadataHeader(s *Writer, block_size uint) { + bw := &s.bw + + bw.writeBits(1, 0) + bw.writeBits(2, 3) + bw.writeBits(1, 0) if block_size == 0 { - writeBits(2, 0, &storage_ix, header) + bw.writeBits(2, 0) } else { var nbits uint32 if block_size == 1 { @@ -1429,95 +901,62 @@ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { nbits = log2FloorNonZero(uint(uint32(block_size)-1)) + 1 } var nbytes uint32 = (nbits + 7) / 8 - writeBits(2, uint64(nbytes), &storage_ix, header) - writeBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header) + bw.writeBits(2, uint64(nbytes)) + bw.writeBits(uint(8*nbytes), uint64(block_size)-1) } - return (storage_ix + 7) >> 3 + bw.jumpToByteBoundary() } func injectBytePaddingBlock(s *Writer) { - var seal uint32 = uint32(s.last_bytes_) - var seal_bits uint = uint(s.last_bytes_bits_) - var destination []byte - s.last_bytes_ = 0 - s.last_bytes_bits_ = 0 - /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */ - seal |= 0x6 << seal_bits - - seal_bits += 6 - - /* If we have already created storage, then append to it. - Storage is valid until next block is being compressed. */ - if s.next_out_ != nil { - destination = s.next_out_[s.available_out_:] - } else { - destination = s.tiny_buf_.u8[:] - s.next_out_ = destination - } - - destination[0] = byte(seal) - if seal_bits > 8 { - destination[1] = byte(seal >> 8) - } - if seal_bits > 16 { - destination[2] = byte(seal >> 16) - } - s.available_out_ += (seal_bits + 7) >> 3 + s.bw.writeBits(6, 0x6) + s.bw.jumpToByteBoundary() + s.writeOutput(s.bw.dst) + s.bw.dst = s.bw.dst[:0] } func checkFlushComplete(s *Writer) { - if s.stream_state_ == streamFlushRequested && s.available_out_ == 0 { + if s.stream_state_ == streamFlushRequested && s.err == nil { s.stream_state_ = streamProcessing - s.next_out_ = nil } } func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte) bool { var block_size_limit uint = uint(1) << s.params.lgwin var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit)) - var tmp_command_buf []uint32 = nil var command_buf []uint32 = nil - var tmp_literal_buf []byte = nil var literal_buf []byte = nil if s.params.quality != fastOnePassCompressionQuality && s.params.quality != fastTwoPassCompressionQuality { return false } if s.params.quality == fastTwoPassCompressionQuality { - if s.command_buf_ == nil && buf_size == kCompressFragmentTwoPassBlockSize { - s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) - s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) - } - - if s.command_buf_ != nil { - command_buf = s.command_buf_ - literal_buf = s.literal_buf_ + if s.command_buf_ == nil || cap(s.command_buf_) < int(buf_size) { + s.command_buf_ = make([]uint32, buf_size) + s.literal_buf_ = make([]byte, buf_size) } else { - tmp_command_buf = make([]uint32, buf_size) - tmp_literal_buf = make([]byte, buf_size) - command_buf = tmp_command_buf - literal_buf = tmp_literal_buf + s.command_buf_ = s.command_buf_[:buf_size] + s.literal_buf_ = s.literal_buf_[:buf_size] } + + command_buf = s.command_buf_ + literal_buf = s.literal_buf_ } for { - if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { injectBytePaddingBlock(s) continue } - /* Compress block only when internal output buffer is empty, stream is not + /* Compress block only when stream is not finished, there is no pending flush request, and there is either additional input or pending operation. */ - if s.available_out_ == 0 && s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { + if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { var block_size uint = brotli_min_size_t(block_size_limit, *available_in) var is_last bool = (*available_in == block_size) && (op == int(operationFinish)) var force_flush bool = (*available_in == block_size) && (op == int(operationFlush)) - var max_out_size uint = 2*block_size + 503 - var storage []byte = nil - var storage_ix uint = uint(s.last_bytes_bits_) var table_size uint var table []int @@ -1526,26 +965,18 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ continue } - storage = getBrotliStorage(s, max_out_size) - - storage[0] = byte(s.last_bytes_) - storage[1] = byte(s.last_bytes_ >> 8) table = getHashTable(s, s.params.quality, block_size, &table_size) if s.params.quality == fastOnePassCompressionQuality { - compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) + compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &s.bw) } else { - compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage) + compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &s.bw) } *next_in = (*next_in)[block_size:] *available_in -= block_size - var out_bytes uint = storage_ix >> 3 - s.next_out_ = storage - s.available_out_ = out_bytes - - s.last_bytes_ = uint16(storage[storage_ix>>3]) - s.last_bytes_bits_ = byte(storage_ix & 7) + s.writeOutput(s.bw.dst) + s.bw.dst = s.bw.dst[:0] if force_flush { s.stream_state_ = streamFlushRequested @@ -1559,8 +990,6 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ break } - tmp_command_buf = nil - tmp_literal_buf = nil checkFlushComplete(s) return true } @@ -1581,17 +1010,13 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { } for { - if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { injectBytePaddingBlock(s) continue } - if s.available_out_ != 0 { - break - } - if s.input_pos_ != s.last_flush_pos_ { - var result bool = encodeData(s, false, true, &s.available_out_, &s.next_out_) + var result bool = encodeData(s, false, true) if !result { return false } @@ -1599,8 +1024,9 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { } if s.stream_state_ == streamMetadataHead { - s.next_out_ = s.tiny_buf_.u8[:] - s.available_out_ = writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.next_out_) + writeMetadataHeader(s, uint(s.remaining_metadata_bytes_)) + s.writeOutput(s.bw.dst) + s.bw.dst = s.bw.dst[:0] s.stream_state_ = streamMetadataBody continue } else { @@ -1614,12 +1040,11 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { /* This guarantees progress in "TakeOutput" workflow. */ var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16) - s.next_out_ = s.tiny_buf_.u8[:] - copy(s.next_out_, (*next_in)[:c]) + copy(s.tiny_buf_.u8[:], (*next_in)[:c]) *next_in = (*next_in)[c:] *available_in -= uint(c) s.remaining_metadata_bytes_ -= c - s.available_out_ = uint(c) + s.writeOutput(s.tiny_buf_.u8[:c]) continue } @@ -1687,20 +1112,20 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt continue } - if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { injectBytePaddingBlock(s) continue } - /* Compress data only when internal output buffer is empty, stream is not + /* Compress data only when stream is not finished and there is no pending flush request. */ - if s.available_out_ == 0 && s.stream_state_ == streamProcessing { + if s.stream_state_ == streamProcessing { if remaining_block_size == 0 || op != int(operationProcess) { var is_last bool = ((*available_in == 0) && op == int(operationFinish)) var force_flush bool = ((*available_in == 0) && op == int(operationFlush)) var result bool updateSizeHint(s, *available_in) - result = encodeData(s, is_last, force_flush, &s.available_out_, &s.next_out_) + result = encodeData(s, is_last, force_flush) if !result { return false } @@ -1721,18 +1146,13 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt return true } -func encoderHasMoreOutput(s *Writer) bool { - return s.available_out_ != 0 -} - -func encoderTakeOutput(s *Writer) []byte { - if s.available_out_ == 0 { - return nil +func (w *Writer) writeOutput(data []byte) { + if w.err != nil { + return } - result := s.next_out_[:s.available_out_] - s.total_out_ += s.available_out_ - s.available_out_ = 0 - checkFlushComplete(s) - return result + _, w.err = w.dst.Write(data) + if w.err == nil { + checkFlushComplete(w) + } } diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode.go b/vendor/github.com/andybalholm/brotli/entropy_encode.go index d0c1dca250a14..3f469a3dd94a5 100644 --- a/vendor/github.com/andybalholm/brotli/entropy_encode.go +++ b/vendor/github.com/andybalholm/brotli/entropy_encode.go @@ -24,7 +24,7 @@ func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) { } /* Input size optimized Shell sort. */ -type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool +type huffmanTreeComparator func(huffmanTree, huffmanTree) bool var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1} @@ -36,14 +36,13 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom var tmp huffmanTree = items[i] var k uint = i var j uint = i - 1 - for comparator(&tmp, &items[j]) { + for comparator(tmp, items[j]) { items[k] = items[j] k = j - tmp10 := j - j-- - if tmp10 == 0 { + if j == 0 { break } + j-- } items[k] = tmp @@ -63,7 +62,7 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom for i = gap; i < n; i++ { var j uint = i var tmp huffmanTree = items[i] - for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap { + for ; j >= gap && comparator(tmp, items[j-gap]); j -= gap { items[j] = items[j-gap] } @@ -105,7 +104,7 @@ func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { } /* Sort the root nodes, least popular first. */ -func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool { +func sortHuffmanTree(v0 huffmanTree, v1 huffmanTree) bool { if v0.total_count_ != v1.total_count_ { return v0.total_count_ < v1.total_count_ } diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode_static.go b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go index 5ddf3fcbaef38..2543f8f07d02c 100644 --- a/vendor/github.com/andybalholm/brotli/entropy_encode_static.go +++ b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go @@ -778,8 +778,9 @@ var kStaticDistanceCodeDepth = [64]byte{ var kCodeLengthBits = [18]uint32{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7} -func storeStaticCodeLengthCode(storage_ix *uint, storage []byte) { - writeBits(40, 0x0000FF55555554, storage_ix, storage) +func storeStaticCodeLengthCode(bw *bitWriter) { + bw.writeBits(32, 0x55555554) + bw.writeBits(8, 0xFF) } var kZeroRepsBits = [numCommandSymbols]uint64{ @@ -4317,9 +4318,10 @@ var kStaticCommandCodeBits = [numCommandSymbols]uint16{ 2047, } -func storeStaticCommandHuffmanTree(storage_ix *uint, storage []byte) { - writeBits(56, 0x92624416307003, storage_ix, storage) - writeBits(3, 0x00000000, storage_ix, storage) +func storeStaticCommandHuffmanTree(bw *bitWriter) { + bw.writeBits(32, 0x16307003) + bw.writeBits(24, 0x926244) + bw.writeBits(3, 0x00000000) } var kStaticDistanceCodeBits = [64]uint16{ @@ -4389,6 +4391,6 @@ var kStaticDistanceCodeBits = [64]uint16{ 63, } -func storeStaticDistanceHuffmanTree(storage_ix *uint, storage []byte) { - writeBits(28, 0x0369DC03, storage_ix, storage) +func storeStaticDistanceHuffmanTree(bw *bitWriter) { + bw.writeBits(28, 0x0369DC03) } diff --git a/vendor/github.com/andybalholm/brotli/find_match_length.go b/vendor/github.com/andybalholm/brotli/find_match_length.go index 14d350aa59a8d..09d2ae6726830 100644 --- a/vendor/github.com/andybalholm/brotli/find_match_length.go +++ b/vendor/github.com/andybalholm/brotli/find_match_length.go @@ -1,5 +1,11 @@ package brotli +import ( + "encoding/binary" + "math/bits" + "runtime" +) + /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -9,6 +15,29 @@ package brotli /* Function to find maximal matching prefixes of strings. */ func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { var matched uint = 0 + _, _ = s1[limit-1], s2[limit-1] // bounds check + switch runtime.GOARCH { + case "amd64": + // Compare 8 bytes at at time. + for matched+8 <= limit { + w1 := binary.LittleEndian.Uint64(s1[matched:]) + w2 := binary.LittleEndian.Uint64(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros64(w1^w2)>>3) + } + matched += 8 + } + case "386": + // Compare 4 bytes at at time. + for matched+4 <= limit { + w1 := binary.LittleEndian.Uint32(s1[matched:]) + w2 := binary.LittleEndian.Uint32(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros32(w1^w2)>>3) + } + matched += 4 + } + } for matched < limit && s1[matched] == s2[matched] { matched++ } diff --git a/vendor/github.com/andybalholm/brotli/histogram.go b/vendor/github.com/andybalholm/brotli/histogram.go index f208ff74b979d..0346622beb310 100644 --- a/vendor/github.com/andybalholm/brotli/histogram.go +++ b/vendor/github.com/andybalholm/brotli/histogram.go @@ -163,7 +163,7 @@ func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) { self.split_ = split self.idx_ = 0 self.type_ = 0 - if split.lengths != nil { + if len(split.lengths) > 0 { self.length_ = uint(split.lengths[0]) } else { self.length_ = 0 @@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) { self.length_-- } -func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { +func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { var pos uint = start_pos var literal_it blockSplitIterator var insert_and_copy_it blockSplitIterator var dist_it blockSplitIterator - var i uint initBlockSplitIterator(&literal_it, literal_split) initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) initBlockSplitIterator(&dist_it, dist_split) - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] var j uint blockSplitIteratorNext(&insert_and_copy_it) diff --git a/vendor/github.com/andybalholm/brotli/memory.go b/vendor/github.com/andybalholm/brotli/memory.go index 7208a3bbf6f09..a07c7050a07f9 100644 --- a/vendor/github.com/andybalholm/brotli/memory.go +++ b/vendor/github.com/andybalholm/brotli/memory.go @@ -23,12 +23,18 @@ func brotli_ensure_capacity_uint8_t(a *[]byte, c *uint, r uint) { for new_size < r { new_size *= 2 } - var new_array []byte = make([]byte, new_size) - if *c != 0 { - copy(new_array, (*a)[:*c]) + + if cap(*a) < int(new_size) { + var new_array []byte = make([]byte, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } + + *a = new_array + } else { + *a = (*a)[:new_size] } - *a = new_array *c = new_size } } @@ -45,12 +51,16 @@ func brotli_ensure_capacity_uint32_t(a *[]uint32, c *uint, r uint) { new_size *= 2 } - new_array = make([]uint32, new_size) - if *c != 0 { - copy(new_array, (*a)[:*c]) - } + if cap(*a) < int(new_size) { + new_array = make([]uint32, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } - *a = new_array + *a = new_array + } else { + *a = (*a)[:new_size] + } *c = new_size } } diff --git a/vendor/github.com/andybalholm/brotli/metablock.go b/vendor/github.com/andybalholm/brotli/metablock.go index 4a412cf4e245f..3014df8cdf100 100644 --- a/vendor/github.com/andybalholm/brotli/metablock.go +++ b/vendor/github.com/andybalholm/brotli/metablock.go @@ -1,5 +1,9 @@ package brotli +import ( + "sync" +) + /* Copyright 2014 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -25,31 +29,30 @@ type metaBlockSplit struct { distance_histograms_size uint } -func initMetaBlockSplit(mb *metaBlockSplit) { - initBlockSplit(&mb.literal_split) - initBlockSplit(&mb.command_split) - initBlockSplit(&mb.distance_split) - mb.literal_context_map = nil - mb.literal_context_map_size = 0 - mb.distance_context_map = nil - mb.distance_context_map_size = 0 - mb.literal_histograms = nil - mb.literal_histograms_size = 0 - mb.command_histograms = nil - mb.command_histograms_size = 0 - mb.distance_histograms = nil - mb.distance_histograms_size = 0 +var metaBlockPool sync.Pool + +func getMetaBlockSplit() *metaBlockSplit { + mb, _ := metaBlockPool.Get().(*metaBlockSplit) + + if mb == nil { + mb = &metaBlockSplit{} + } else { + initBlockSplit(&mb.literal_split) + initBlockSplit(&mb.command_split) + initBlockSplit(&mb.distance_split) + mb.literal_context_map = mb.literal_context_map[:0] + mb.literal_context_map_size = 0 + mb.distance_context_map = mb.distance_context_map[:0] + mb.distance_context_map_size = 0 + mb.literal_histograms = mb.literal_histograms[:0] + mb.command_histograms = mb.command_histograms[:0] + mb.distance_histograms = mb.distance_histograms[:0] + } + return mb } -func destroyMetaBlockSplit(mb *metaBlockSplit) { - destroyBlockSplit(&mb.literal_split) - destroyBlockSplit(&mb.command_split) - destroyBlockSplit(&mb.distance_split) - mb.literal_context_map = nil - mb.distance_context_map = nil - mb.literal_histograms = nil - mb.command_histograms = nil - mb.distance_histograms = nil +func freeMetaBlockSplit(mb *metaBlockSplit) { + metaBlockPool.Put(mb) } func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { @@ -84,14 +87,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) dist_params.max_distance = uint(max_distance) } -func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) { - var i uint - +func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) { if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { return } - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) @@ -99,8 +100,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d } } -func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { - var i uint +func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { var equal_params bool = false var dist_prefix uint16 var dist_extra uint32 @@ -112,8 +112,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc equal_params = true } - for i = 0; i < num_commands; i++ { - var cmd *command = &cmds[i] + for i := range cmds { + cmd := &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if equal_params { dist_prefix = cmd.dist_prefix_ @@ -137,7 +137,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc var buildMetaBlock_kMaxNumberOfHistograms uint = 256 -func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) { +func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) { var distance_histograms []histogramDistance var literal_histograms []histogramLiteral var literal_context_modes []int = nil @@ -164,7 +164,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam check_orig = false } - skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) + skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost) if skip || (dist_cost > best_dist_cost) { break } @@ -181,7 +181,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam if check_orig { var dist_cost float64 - computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) + computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost) if dist_cost < best_dist_cost { /* NB: currently unused; uncomment when more param tuning is added. */ /* best_dist_cost = dist_cost; */ @@ -189,9 +189,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam } } - recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist) + recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist) - splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) + splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) if !params.disable_literal_context_modeling { literal_context_multiplier = 1 << literalContextBits @@ -209,21 +209,30 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam distance_histograms = make([]histogramDistance, distance_histograms_size) clearHistogramsDistance(distance_histograms, distance_histograms_size) - assert(mb.command_histograms == nil) mb.command_histograms_size = mb.command_split.num_types - mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) + if cap(mb.command_histograms) < int(mb.command_histograms_size) { + mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) + } else { + mb.command_histograms = mb.command_histograms[:mb.command_histograms_size] + } clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) - buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) + buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) literal_context_modes = nil - assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits - mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } - assert(mb.literal_histograms == nil) mb.literal_histograms_size = mb.literal_context_map_size - mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) + if cap(mb.literal_histograms) < int(mb.literal_histograms_size) { + mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) + } else { + mb.literal_histograms = mb.literal_histograms[:mb.literal_histograms_size] + } clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map) literal_histograms = nil @@ -239,13 +248,19 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam } } - assert(mb.distance_context_map == nil) mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits - mb.distance_context_map = make([]uint32, (mb.distance_context_map_size)) + if cap(mb.distance_context_map) < int(mb.distance_context_map_size) { + mb.distance_context_map = make([]uint32, (mb.distance_context_map_size)) + } else { + mb.distance_context_map = mb.distance_context_map[:mb.distance_context_map_size] + } - assert(mb.distance_histograms == nil) mb.distance_histograms_size = mb.distance_context_map_size - mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size)) + if cap(mb.distance_histograms) < int(mb.distance_histograms_size) { + mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size)) + } else { + mb.distance_histograms = mb.distance_histograms[:mb.distance_histograms_size] + } clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map) distance_histograms = nil @@ -298,9 +313,12 @@ func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, nu brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) split.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types * num_contexts - *histograms = make([]histogramLiteral, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramLiteral, (*histograms_size)) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ @@ -453,9 +471,12 @@ func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, cont func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) { var i uint - assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits - mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } for i = 0; i < mb.literal_split.num_types; i++ { var offset uint32 = uint32(i * num_contexts) @@ -466,7 +487,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB } } -func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { var lit_blocks struct { plain blockSplitterLiteral ctx contextBlockSplitter @@ -474,8 +495,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b var cmd_blocks blockSplitterCommand var dist_blocks blockSplitterDistance var num_literals uint = 0 - var i uint - for i = 0; i < n_commands; i++ { + for i := range commands { num_literals += uint(commands[i].insert_len_) } @@ -485,11 +505,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) } - initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) - initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) + initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) + initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -530,11 +549,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b } } -func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { if num_contexts == 1 { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb) } else { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb) } } diff --git a/vendor/github.com/andybalholm/brotli/metablock_command.go b/vendor/github.com/andybalholm/brotli/metablock_command.go index d47541c5e6aa0..14c7b77135dab 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_command.go +++ b/vendor/github.com/andybalholm/brotli/metablock_command.go @@ -43,9 +43,12 @@ func initBlockSplitterCommand(self *blockSplitterCommand, alphabet_size uint, mi brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramCommand, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramCommand, (*histograms_size)) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/metablock_distance.go b/vendor/github.com/andybalholm/brotli/metablock_distance.go index 95923127a6dcf..5110a810e96de 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_distance.go +++ b/vendor/github.com/andybalholm/brotli/metablock_distance.go @@ -43,9 +43,12 @@ func initBlockSplitterDistance(self *blockSplitterDistance, alphabet_size uint, brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramDistance, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramDistance, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/metablock_literal.go b/vendor/github.com/andybalholm/brotli/metablock_literal.go index d7e8a7c923321..307f8da88f48f 100644 --- a/vendor/github.com/andybalholm/brotli/metablock_literal.go +++ b/vendor/github.com/andybalholm/brotli/metablock_literal.go @@ -43,9 +43,12 @@ func initBlockSplitterLiteral(self *blockSplitterLiteral, alphabet_size uint, mi brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) self.split_.num_blocks = max_num_blocks - assert(*histograms == nil) *histograms_size = max_num_types - *histograms = make([]histogramLiteral, (*histograms_size)) + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramLiteral, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } self.histograms_ = *histograms /* Clear only current histogram. */ diff --git a/vendor/github.com/andybalholm/brotli/ringbuffer.go b/vendor/github.com/andybalholm/brotli/ringbuffer.go index 693a3f65d3a46..1c8f86feece36 100644 --- a/vendor/github.com/andybalholm/brotli/ringbuffer.go +++ b/vendor/github.com/andybalholm/brotli/ringbuffer.go @@ -27,10 +27,7 @@ type ringBuffer struct { } func ringBufferInit(rb *ringBuffer) { - rb.cur_size_ = 0 rb.pos_ = 0 - rb.data_ = nil - rb.buffer_ = nil } func ringBufferSetup(params *encoderParams, rb *ringBuffer) { @@ -47,11 +44,16 @@ const kSlackForEightByteHashingEverywhere uint = 7 /* Allocates or re-allocates data_ to the given length + plus some slack region before and after. Fills the slack regions with zeros. */ func ringBufferInitBuffer(buflen uint32, rb *ringBuffer) { - var new_data []byte = make([]byte, (2 + uint(buflen) + kSlackForEightByteHashingEverywhere)) + var new_data []byte var i uint + size := 2 + int(buflen) + int(kSlackForEightByteHashingEverywhere) + if cap(rb.data_) < size { + new_data = make([]byte, size) + } else { + new_data = rb.data_[:size] + } if rb.data_ != nil { copy(new_data, rb.data_[:2+rb.cur_size_+uint32(kSlackForEightByteHashingEverywhere)]) - rb.data_ = nil } rb.data_ = new_data diff --git a/vendor/github.com/andybalholm/brotli/write_bits.go b/vendor/github.com/andybalholm/brotli/write_bits.go index 8f15c20257fce..2d216d7ccd92f 100644 --- a/vendor/github.com/andybalholm/brotli/write_bits.go +++ b/vendor/github.com/andybalholm/brotli/write_bits.go @@ -8,49 +8,87 @@ package brotli /* Write bits into a byte array. */ -/* This function writes bits into bytes in increasing addresses, and within - a byte least-significant-bit first. - - The function can write up to 56 bits in one go with WriteBits - Example: let's assume that 3 bits (Rs below) have been written already: - - BYTE-0 BYTE+1 BYTE+2 - - 0000 0RRR 0000 0000 0000 0000 - - Now, we could write 5 or less bits in MSB by just sifting by 3 - and OR'ing to BYTE-0. - - For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, - and locate the rest in BYTE+1, BYTE+2, etc. */ -func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { - var array_pos []byte = array[*pos>>3:] - var bits_reserved_in_first_byte uint = (*pos & 7) - /* implicit & 0xFF is assumed for uint8_t arithmetics */ - - var bits_left_to_write uint - bits <<= bits_reserved_in_first_byte - array_pos[0] |= byte(bits) - array_pos = array_pos[1:] - for bits_left_to_write = n_bits + bits_reserved_in_first_byte; bits_left_to_write >= 9; bits_left_to_write -= 8 { - bits >>= 8 - array_pos[0] = byte(bits) - array_pos = array_pos[1:] - } +type bitWriter struct { + dst []byte + + // Data waiting to be written is the low nbits of bits. + bits uint64 + nbits uint +} - array_pos[0] = 0 - *pos += n_bits +func (w *bitWriter) writeBits(nb uint, b uint64) { + w.bits |= b << w.nbits + w.nbits += nb + if w.nbits >= 32 { + bits := w.bits + w.bits >>= 32 + w.nbits -= 32 + w.dst = append(w.dst, + byte(bits), + byte(bits>>8), + byte(bits>>16), + byte(bits>>24), + ) + } } -func writeSingleBit(bit bool, pos *uint, array []byte) { +func (w *bitWriter) writeSingleBit(bit bool) { if bit { - writeBits(1, 1, pos, array) + w.writeBits(1, 1) } else { - writeBits(1, 0, pos, array) + w.writeBits(1, 0) + } +} + +func (w *bitWriter) jumpToByteBoundary() { + dst := w.dst + for w.nbits != 0 { + dst = append(dst, byte(w.bits)) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + } + w.bits = 0 + w.dst = dst +} + +func (w *bitWriter) writeBytes(b []byte) { + if w.nbits&7 != 0 { + panic("writeBytes with unfinished bits") } + for w.nbits != 0 { + w.dst = append(w.dst, byte(w.bits)) + w.bits >>= 8 + w.nbits -= 8 + } + w.dst = append(w.dst, b...) +} + +func (w *bitWriter) getPos() uint { + return uint(len(w.dst)<<3) + w.nbits +} + +func (w *bitWriter) rewind(p uint) { + w.bits = uint64(w.dst[p>>3] & byte((1<<(p&7))-1)) + w.nbits = p & 7 + w.dst = w.dst[:p>>3] } -func writeBitsPrepareStorage(pos uint, array []byte) { - assert(pos&7 == 0) - array[pos>>3] = 0 +func (w *bitWriter) updateBits(n_bits uint, bits uint32, pos uint) { + for n_bits > 0 { + var byte_pos uint = pos >> 3 + var n_unchanged_bits uint = pos & 7 + var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) + var total_bits uint = n_unchanged_bits + n_changed_bits + var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) + var unchanged_bits uint32 = uint32(w.dst[byte_pos]) & mask + var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) + w.dst[byte_pos] = byte(changed_bits<>= n_changed_bits + pos += n_changed_bits + } } diff --git a/vendor/github.com/andybalholm/brotli/writer.go b/vendor/github.com/andybalholm/brotli/writer.go index ec333f9cff6fc..63676b4673794 100644 --- a/vendor/github.com/andybalholm/brotli/writer.go +++ b/vendor/github.com/andybalholm/brotli/writer.go @@ -67,6 +67,9 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { if w.dst == nil { return 0, errWriterClosed } + if w.err != nil { + return 0, w.err + } for { availableIn := uint(len(p)) @@ -79,16 +82,8 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { return n, errEncode } - outputData := encoderTakeOutput(w) - - if len(outputData) > 0 { - _, err = w.dst.Write(outputData) - if err != nil { - return n, err - } - } - if len(p) == 0 { - return n, nil + if len(p) == 0 || w.err != nil { + return n, w.err } } } diff --git a/vendor/github.com/dsnet/compress/go.mod b/vendor/github.com/dsnet/compress/go.mod index 7a0bc001eb207..304df87b51b73 100644 --- a/vendor/github.com/dsnet/compress/go.mod +++ b/vendor/github.com/dsnet/compress/go.mod @@ -4,7 +4,8 @@ go 1.9 require ( github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 + github.com/google/go-cmp v0.5.5 // indirect github.com/klauspost/compress v1.4.1 github.com/klauspost/cpuid v1.2.0 // indirect - github.com/ulikunitz/xz v0.5.6 + github.com/ulikunitz/xz v0.5.8 ) diff --git a/vendor/github.com/dsnet/compress/go.sum b/vendor/github.com/dsnet/compress/go.sum index b6fd40c7983ed..936464a87b981 100644 --- a/vendor/github.com/dsnet/compress/go.sum +++ b/vendor/github.com/dsnet/compress/go.sum @@ -1,8 +1,12 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go index 6d4c1e98bc5f9..4a73e1bdd30a3 100644 --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -127,7 +127,7 @@ func (e *fastGen) addBlock(src []byte) int32 { // hash4 returns the hash of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <32. func hash4u(u uint32, h uint8) uint32 { - return (u * prime4bytes) >> ((32 - h) & 31) + return (u * prime4bytes) >> ((32 - h) & reg8SizeMask32) } type tableEntryPrev struct { @@ -138,25 +138,25 @@ type tableEntryPrev struct { // hash4x64 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <32. func hash4x64(u uint64, h uint8) uint32 { - return (uint32(u) * prime4bytes) >> ((32 - h) & 31) + return (uint32(u) * prime4bytes) >> ((32 - h) & reg8SizeMask32) } // hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <64. func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) + return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & reg8SizeMask64)) } // hash8 returns the hash of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <64. func hash8(u uint64, h uint8) uint32 { - return uint32((u * prime8bytes) >> ((64 - h) & 63)) + return uint32((u * prime8bytes) >> ((64 - h) & reg8SizeMask64)) } // hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <64. func hash6(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) + return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & reg8SizeMask64)) } // matchlen will return the match length between offsets and t in src. diff --git a/vendor/github.com/klauspost/compress/flate/gen_inflate.go b/vendor/github.com/klauspost/compress/flate/gen_inflate.go index c74a95fe7f6f3..35fc072a3efa3 100644 --- a/vendor/github.com/klauspost/compress/flate/gen_inflate.go +++ b/vendor/github.com/klauspost/compress/flate/gen_inflate.go @@ -42,16 +42,6 @@ func (f *decompressor) $FUNCNAME$() { stateDict ) fr := f.r.($TYPE$) - moreBits := func() error { - c, err := fr.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil - } switch f.stepState { case stateInit: @@ -85,7 +75,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -104,7 +94,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -112,9 +102,7 @@ readLiteral: } } - var n uint // number of bits extra var length int - var err error switch { case v < 256: f.dict.writeByte(byte(v)) @@ -131,68 +119,96 @@ readLiteral: // otherwise, reference to older data case v < 265: length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 case v < maxNumLit: - length = 258 - n = 0 - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) for f.nb < n { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits n>0:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + return } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb<5:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { - if debugDecode { - fmt.Println("huffsym:", err) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := fr.ReadByte() + if err != nil { + f.b = b + f.nb = nb + f.err = noEOF(err) + return + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := f.hd.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + dist = uint32(chunk >> huffmanValueShift) + break } - f.err = err - return } } @@ -202,20 +218,24 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -225,7 +245,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -233,7 +253,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go index 53fe1d06e25ad..208d66711df86 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -206,7 +206,7 @@ func (w *huffmanBitWriter) write(b []byte) { } func (w *huffmanBitWriter) writeBits(b int32, nb uint16) { - w.bits |= uint64(b) << (w.nbits & 63) + w.bits |= uint64(b) << (w.nbits & reg16SizeMask64) w.nbits += nb if w.nbits >= 48 { w.writeOutBits() @@ -759,7 +759,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) } else { // inlined c := lengths[lengthCode&31] - w.bits |= uint64(c.code) << (w.nbits & 63) + w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64) w.nbits += c.len if w.nbits >= 48 { w.writeOutBits() @@ -779,7 +779,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) } else { // inlined c := offs[offsetCode&31] - w.bits |= uint64(c.code) << (w.nbits & 63) + w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64) w.nbits += c.len if w.nbits >= 48 { w.writeOutBits() @@ -878,7 +878,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { for _, t := range input { // Bitwriting inlined, ~30% speedup c := encoding[t] - w.bits |= uint64(c.code) << ((w.nbits) & 63) + w.bits |= uint64(c.code) << ((w.nbits) & reg16SizeMask64) w.nbits += c.len if w.nbits >= 48 { bits := w.bits diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 3e4259f157bfa..16bc51408e072 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -29,6 +29,13 @@ const ( debugDecode = false ) +// Value of length - 3 and extra bits. +type lengthExtra struct { + length, extra uint8 +} + +var decCodeToLen = [32]lengthExtra{{length: 0x0, extra: 0x0}, {length: 0x1, extra: 0x0}, {length: 0x2, extra: 0x0}, {length: 0x3, extra: 0x0}, {length: 0x4, extra: 0x0}, {length: 0x5, extra: 0x0}, {length: 0x6, extra: 0x0}, {length: 0x7, extra: 0x0}, {length: 0x8, extra: 0x1}, {length: 0xa, extra: 0x1}, {length: 0xc, extra: 0x1}, {length: 0xe, extra: 0x1}, {length: 0x10, extra: 0x2}, {length: 0x14, extra: 0x2}, {length: 0x18, extra: 0x2}, {length: 0x1c, extra: 0x2}, {length: 0x20, extra: 0x3}, {length: 0x28, extra: 0x3}, {length: 0x30, extra: 0x3}, {length: 0x38, extra: 0x3}, {length: 0x40, extra: 0x4}, {length: 0x50, extra: 0x4}, {length: 0x60, extra: 0x4}, {length: 0x70, extra: 0x4}, {length: 0x80, extra: 0x5}, {length: 0xa0, extra: 0x5}, {length: 0xc0, extra: 0x5}, {length: 0xe0, extra: 0x5}, {length: 0xff, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}} + // Initialize the fixedHuffmanDecoder only once upon first use. var fixedOnce sync.Once var fixedHuffmanDecoder huffmanDecoder @@ -522,8 +529,8 @@ func (f *decompressor) readHuffman() error { return err } } - rep += int(f.b & uint32(1<>= nb + rep += int(f.b & uint32(1<<(nb®SizeMaskUint32)-1)) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb if i+rep > n { if debugDecode { @@ -603,7 +610,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -622,7 +629,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -685,12 +692,12 @@ readLiteral: return } } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { if err = f.moreBits(); err != nil { @@ -701,17 +708,19 @@ readLiteral: return } } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { + sym, err := f.huffSym(f.hd) + if err != nil { if debugDecode { fmt.Println("huffsym:", err) } f.err = err return } + dist = uint32(sym) } switch { @@ -720,7 +729,7 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { if err = f.moreBits(); err != nil { if debugDecode { @@ -730,10 +739,10 @@ readLiteral: return } } - extra |= int(f.b & uint32(1<>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -743,7 +752,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -751,7 +760,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } @@ -869,7 +878,7 @@ func (f *decompressor) moreBits() error { return noEOF(err) } f.roffset++ - f.b |= uint32(c) << f.nb + f.b |= uint32(c) << (f.nb & regSizeMaskUint32) f.nb += 8 return nil } @@ -894,7 +903,7 @@ func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { return 0, noEOF(err) } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := h.chunks[b&(huffmanNumChunks-1)] @@ -913,7 +922,7 @@ func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { f.err = CorruptInputError(f.roffset) return 0, f.err } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n return int(chunk >> huffmanValueShift), nil } diff --git a/vendor/github.com/klauspost/compress/flate/inflate_gen.go b/vendor/github.com/klauspost/compress/flate/inflate_gen.go index 397dc1b1a134b..cc6db27925ce1 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go +++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go @@ -20,16 +20,6 @@ func (f *decompressor) huffmanBytesBuffer() { stateDict ) fr := f.r.(*bytes.Buffer) - moreBits := func() error { - c, err := fr.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil - } switch f.stepState { case stateInit: @@ -63,7 +53,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -82,7 +72,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -90,9 +80,7 @@ readLiteral: } } - var n uint // number of bits extra var length int - var err error switch { case v < 256: f.dict.writeByte(byte(v)) @@ -109,68 +97,96 @@ readLiteral: // otherwise, reference to older data case v < 265: length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 case v < maxNumLit: - length = 258 - n = 0 - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) for f.nb < n { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits n>0:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + return } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb<5:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { - if debugDecode { - fmt.Println("huffsym:", err) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := fr.ReadByte() + if err != nil { + f.b = b + f.nb = nb + f.err = noEOF(err) + return + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := f.hd.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + dist = uint32(chunk >> huffmanValueShift) + break } - f.err = err - return } } @@ -180,20 +196,24 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -203,7 +223,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -211,7 +231,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } @@ -244,16 +264,6 @@ func (f *decompressor) huffmanBytesReader() { stateDict ) fr := f.r.(*bytes.Reader) - moreBits := func() error { - c, err := fr.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil - } switch f.stepState { case stateInit: @@ -287,7 +297,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -306,7 +316,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -314,9 +324,7 @@ readLiteral: } } - var n uint // number of bits extra var length int - var err error switch { case v < 256: f.dict.writeByte(byte(v)) @@ -333,68 +341,96 @@ readLiteral: // otherwise, reference to older data case v < 265: length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 case v < maxNumLit: - length = 258 - n = 0 - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) for f.nb < n { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits n>0:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + return } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb<5:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { - if debugDecode { - fmt.Println("huffsym:", err) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := fr.ReadByte() + if err != nil { + f.b = b + f.nb = nb + f.err = noEOF(err) + return + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := f.hd.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + dist = uint32(chunk >> huffmanValueShift) + break } - f.err = err - return } } @@ -404,20 +440,24 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -427,7 +467,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -435,7 +475,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } @@ -468,16 +508,6 @@ func (f *decompressor) huffmanBufioReader() { stateDict ) fr := f.r.(*bufio.Reader) - moreBits := func() error { - c, err := fr.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil - } switch f.stepState { case stateInit: @@ -511,7 +541,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -530,7 +560,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -538,9 +568,7 @@ readLiteral: } } - var n uint // number of bits extra var length int - var err error switch { case v < 256: f.dict.writeByte(byte(v)) @@ -557,68 +585,96 @@ readLiteral: // otherwise, reference to older data case v < 265: length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 case v < maxNumLit: - length = 258 - n = 0 - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) for f.nb < n { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits n>0:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + return } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb<5:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { - if debugDecode { - fmt.Println("huffsym:", err) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := fr.ReadByte() + if err != nil { + f.b = b + f.nb = nb + f.err = noEOF(err) + return + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := f.hd.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + dist = uint32(chunk >> huffmanValueShift) + break } - f.err = err - return } } @@ -628,20 +684,24 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -651,7 +711,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -659,7 +719,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } @@ -692,16 +752,6 @@ func (f *decompressor) huffmanStringsReader() { stateDict ) fr := f.r.(*strings.Reader) - moreBits := func() error { - c, err := fr.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << f.nb - f.nb += 8 - return nil - } switch f.stepState { case stateInit: @@ -735,7 +785,7 @@ readLiteral: return } f.roffset++ - b |= uint32(c) << (nb & 31) + b |= uint32(c) << (nb & regSizeMaskUint32) nb += 8 } chunk := f.hl.chunks[b&(huffmanNumChunks-1)] @@ -754,7 +804,7 @@ readLiteral: f.err = CorruptInputError(f.roffset) return } - f.b = b >> (n & 31) + f.b = b >> (n & regSizeMaskUint32) f.nb = nb - n v = int(chunk >> huffmanValueShift) break @@ -762,9 +812,7 @@ readLiteral: } } - var n uint // number of bits extra var length int - var err error switch { case v < 256: f.dict.writeByte(byte(v)) @@ -781,68 +829,96 @@ readLiteral: // otherwise, reference to older data case v < 265: length = v - (257 - 3) - n = 0 - case v < 269: - length = v*2 - (265*2 - 11) - n = 1 - case v < 273: - length = v*4 - (269*4 - 19) - n = 2 - case v < 277: - length = v*8 - (273*8 - 35) - n = 3 - case v < 281: - length = v*16 - (277*16 - 67) - n = 4 - case v < 285: - length = v*32 - (281*32 - 131) - n = 5 case v < maxNumLit: - length = 258 - n = 0 - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - return - } - if n > 0 { + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) for f.nb < n { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits n>0:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - length += int(f.b & uint32(1<>= n + length += int(f.b & uint32(1<<(n®SizeMaskUint32)-1)) + f.b >>= n & regSizeMaskUint32 f.nb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + return } - var dist int + var dist uint32 if f.hd == nil { for f.nb < 5 { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb<5:", err) } f.err = err return } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 } - dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3))) + dist = uint32(bits.Reverse8(uint8(f.b & 0x1F << 3))) f.b >>= 5 f.nb -= 5 } else { - if dist, err = f.huffSym(f.hd); err != nil { - if debugDecode { - fmt.Println("huffsym:", err) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := fr.ReadByte() + if err != nil { + f.b = b + f.nb = nb + f.err = noEOF(err) + return + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := f.hd.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + dist = uint32(chunk >> huffmanValueShift) + break } - f.err = err - return } } @@ -852,20 +928,24 @@ readLiteral: case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << nb + extra := (dist & 1) << (nb & regSizeMaskUint32) for f.nb < nb { - if err = moreBits(); err != nil { + c, err := fr.ReadByte() + if err != nil { if debugDecode { fmt.Println("morebits f.nb>= nb + extra |= f.b & uint32(1<<(nb®SizeMaskUint32)-1) + f.b >>= nb & regSizeMaskUint32 f.nb -= nb - dist = 1<<(nb+1) + 1 + extra + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra default: if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -875,7 +955,7 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > f.dict.histSize() { + if dist > uint32(f.dict.histSize()) { if debugDecode { fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) } @@ -883,7 +963,7 @@ readLiteral: return } - f.copyLen, f.copyDist = length, dist + f.copyLen, f.copyDist = length, int(dist) goto copyHistory } diff --git a/vendor/github.com/klauspost/compress/flate/regmask_amd64.go b/vendor/github.com/klauspost/compress/flate/regmask_amd64.go new file mode 100644 index 0000000000000..6ed28061b2b82 --- /dev/null +++ b/vendor/github.com/klauspost/compress/flate/regmask_amd64.go @@ -0,0 +1,37 @@ +package flate + +const ( + // Masks for shifts with register sizes of the shift value. + // This can be used to work around the x86 design of shifting by mod register size. + // It can be used when a variable shift is always smaller than the register size. + + // reg8SizeMaskX - shift value is 8 bits, shifted is X + reg8SizeMask8 = 7 + reg8SizeMask16 = 15 + reg8SizeMask32 = 31 + reg8SizeMask64 = 63 + + // reg16SizeMaskX - shift value is 16 bits, shifted is X + reg16SizeMask8 = reg8SizeMask8 + reg16SizeMask16 = reg8SizeMask16 + reg16SizeMask32 = reg8SizeMask32 + reg16SizeMask64 = reg8SizeMask64 + + // reg32SizeMaskX - shift value is 32 bits, shifted is X + reg32SizeMask8 = reg8SizeMask8 + reg32SizeMask16 = reg8SizeMask16 + reg32SizeMask32 = reg8SizeMask32 + reg32SizeMask64 = reg8SizeMask64 + + // reg64SizeMaskX - shift value is 64 bits, shifted is X + reg64SizeMask8 = reg8SizeMask8 + reg64SizeMask16 = reg8SizeMask16 + reg64SizeMask32 = reg8SizeMask32 + reg64SizeMask64 = reg8SizeMask64 + + // regSizeMaskUintX - shift value is uint, shifted is X + regSizeMaskUint8 = reg8SizeMask8 + regSizeMaskUint16 = reg8SizeMask16 + regSizeMaskUint32 = reg8SizeMask32 + regSizeMaskUint64 = reg8SizeMask64 +) diff --git a/vendor/github.com/klauspost/compress/flate/regmask_other.go b/vendor/github.com/klauspost/compress/flate/regmask_other.go new file mode 100644 index 0000000000000..f477a5d6e5a6f --- /dev/null +++ b/vendor/github.com/klauspost/compress/flate/regmask_other.go @@ -0,0 +1,39 @@ +//+build !amd64 + +package flate + +const ( + // Masks for shifts with register sizes of the shift value. + // This can be used to work around the x86 design of shifting by mod register size. + // It can be used when a variable shift is always smaller than the register size. + + // reg8SizeMaskX - shift value is 8 bits, shifted is X + reg8SizeMask8 = 0xff + reg8SizeMask16 = 0xff + reg8SizeMask32 = 0xff + reg8SizeMask64 = 0xff + + // reg16SizeMaskX - shift value is 16 bits, shifted is X + reg16SizeMask8 = 0xffff + reg16SizeMask16 = 0xffff + reg16SizeMask32 = 0xffff + reg16SizeMask64 = 0xffff + + // reg32SizeMaskX - shift value is 32 bits, shifted is X + reg32SizeMask8 = 0xffffffff + reg32SizeMask16 = 0xffffffff + reg32SizeMask32 = 0xffffffff + reg32SizeMask64 = 0xffffffff + + // reg64SizeMaskX - shift value is 64 bits, shifted is X + reg64SizeMask8 = 0xffffffffffffffff + reg64SizeMask16 = 0xffffffffffffffff + reg64SizeMask32 = 0xffffffffffffffff + reg64SizeMask64 = 0xffffffffffffffff + + // regSizeMaskUintX - shift value is uint, shifted is X + regSizeMaskUint8 = ^uint(0) + regSizeMaskUint16 = ^uint(0) + regSizeMaskUint32 = ^uint(0) + regSizeMaskUint64 = ^uint(0) +) diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index bda4021efd3a5..6bce4e87d4ff6 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -43,6 +43,11 @@ func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { func (b *bitWriter) encSymbol(ct cTable, symbol byte) { enc := ct[symbol] b.bitContainer |= uint64(enc.val) << (b.nBits & 63) + if false { + if enc.nBits == 0 { + panic("nbits 0") + } + } b.nBits += enc.nBits } @@ -54,6 +59,14 @@ func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { sh := b.nBits & 63 combined := uint64(encA.val) | (uint64(encB.val) << (encA.nBits & 63)) b.bitContainer |= combined << sh + if false { + if encA.nBits == 0 { + panic("nbitsA 0") + } + if encB.nBits == 0 { + panic("nbitsB 0") + } + } b.nBits += encA.nBits + encB.nBits } diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 0843cb014ffce..f9ed5f8306ec8 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -77,8 +77,11 @@ func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error) // Each symbol present maximum once or too well distributed. return nil, false, ErrIncompressible } - - if s.Reuse == ReusePolicyPrefer && canReuse { + if s.Reuse == ReusePolicyMust && !canReuse { + // We must reuse, but we can't. + return nil, false, ErrIncompressible + } + if (s.Reuse == ReusePolicyPrefer || s.Reuse == ReusePolicyMust) && canReuse { keepTable := s.cTable keepTL := s.actualTableLog s.cTable = s.prevTable @@ -90,6 +93,9 @@ func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error) s.OutData = s.Out return s.Out, true, nil } + if s.Reuse == ReusePolicyMust { + return nil, false, ErrIncompressible + } // Do not attempt to re-use later. s.prevTable = s.prevTable[:0] } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index a03b2634afb42..41703bba4d65b 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -32,7 +32,7 @@ const use8BitTables = true // The size of the input may be larger than the table definition. // Any content remaining after the table definition will be returned. // If no Scratch is provided a new one is allocated. -// The returned Scratch can be used for decoding input using this table. +// The returned Scratch can be used for encoding or decoding input using this table. func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { s, err = s.prepare(in) if err != nil { @@ -58,8 +58,8 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { s.symbolLen = uint16(oSize) in = in[iSize:] } else { - if len(in) <= int(iSize) { - return s, nil, errors.New("input too small for table") + if len(in) < int(iSize) { + return s, nil, fmt.Errorf("input too small for table, want %d bytes, have %d", iSize, len(in)) } // FSE compressed weights s.fse.DecompressLimit = 255 @@ -138,15 +138,33 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { if len(s.dt.single) != tSize { s.dt.single = make([]dEntrySingle, tSize) } + cTable := s.prevTable + if cap(cTable) < maxSymbolValue+1 { + cTable = make([]cTableEntry, 0, maxSymbolValue+1) + } + cTable = cTable[:maxSymbolValue+1] + s.prevTable = cTable[:s.symbolLen] + s.prevTableLog = s.actualTableLog + for n, w := range s.huffWeight[:s.symbolLen] { if w == 0 { + cTable[n] = cTableEntry{ + val: 0, + nBits: 0, + } continue } length := (uint32(1) << w) >> 1 d := dEntrySingle{ entry: uint16(s.actualTableLog+1-w) | (uint16(n) << 8), } + rank := &rankStats[w] + cTable[n] = cTableEntry{ + val: uint16(*rank >> (w - 1)), + nBits: uint8(d.entry), + } + single := s.dt.single[*rank : *rank+length] for i := range single { single[i] = d diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 177d6c4ea0e54..7ec2022b6506f 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -55,6 +55,9 @@ const ( // ReusePolicyNone will disable re-use of tables. // This is slightly faster than ReusePolicyAllow but may produce larger output. ReusePolicyNone + + // ReusePolicyMust must allow reuse and produce smaller output. + ReusePolicyMust ) type Scratch struct { @@ -116,6 +119,16 @@ type Scratch struct { huffWeight [maxSymbolValue + 1]byte } +// TransferCTable will transfer the previously used compression table. +func (s *Scratch) TransferCTable(src *Scratch) { + if cap(s.prevTable) < len(src.prevTable) { + s.prevTable = make(cTable, 0, maxSymbolValue+1) + } + s.prevTable = s.prevTable[:len(src.prevTable)] + copy(s.prevTable, src.prevTable) + s.prevTableLog = src.prevTableLog +} + func (s *Scratch) prepare(in []byte) (*Scratch, error) { if len(in) > BlockSizeMax { return nil, ErrTooBig diff --git a/vendor/github.com/klauspost/compress/zip/reader.go b/vendor/github.com/klauspost/compress/zip/reader.go new file mode 100644 index 0000000000000..9fab6e38783b8 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/reader.go @@ -0,0 +1,623 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "os" + "time" +) + +var ( + ErrFormat = errors.New("zip: not a valid zip file") + ErrAlgorithm = errors.New("zip: unsupported compression algorithm") + ErrChecksum = errors.New("zip: checksum error") +) + +type Reader struct { + r io.ReaderAt + File []*File + Comment string + decompressors map[uint16]Decompressor +} + +type ReadCloser struct { + f *os.File + Reader +} + +type File struct { + FileHeader + zip *Reader + zipr io.ReaderAt + zipsize int64 + headerOffset int64 +} + +func (f *File) hasDataDescriptor() bool { + return f.Flags&0x8 != 0 +} + +// OpenReader will open the Zip file specified by name and return a ReadCloser. +func OpenReader(name string) (*ReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + r := new(ReadCloser) + if err := r.init(f, fi.Size()); err != nil { + f.Close() + return nil, err + } + r.f = f + return r, nil +} + +// NewReader returns a new Reader reading from r, which is assumed to +// have the given size in bytes. +func NewReader(r io.ReaderAt, size int64) (*Reader, error) { + if size < 0 { + return nil, errors.New("zip: size cannot be negative") + } + zr := new(Reader) + if err := zr.init(r, size); err != nil { + return nil, err + } + return zr, nil +} + +func (z *Reader) init(r io.ReaderAt, size int64) error { + end, err := readDirectoryEnd(r, size) + if err != nil { + return err + } + if end.directoryRecords > uint64(size)/fileHeaderLen { + return fmt.Errorf("archive/zip: TOC declares impossible %d files in %d byte zip", end.directoryRecords, size) + } + z.r = r + z.File = make([]*File, 0, end.directoryRecords) + z.Comment = end.comment + rs := io.NewSectionReader(r, 0, size) + if _, err = rs.Seek(int64(end.directoryOffset), io.SeekStart); err != nil { + return err + } + buf := bufio.NewReader(rs) + + // The count of files inside a zip is truncated to fit in a uint16. + // Gloss over this by reading headers until we encounter + // a bad one, and then only report an ErrFormat or UnexpectedEOF if + // the file count modulo 65536 is incorrect. + for { + f := &File{zip: z, zipr: r, zipsize: size} + err = readDirectoryHeader(f, buf) + if err == ErrFormat || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return err + } + z.File = append(z.File, f) + } + if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here + // Return the readDirectoryHeader error if we read + // the wrong number of directory entries. + return err + } + return nil +} + +// RegisterDecompressor registers or overrides a custom decompressor for a +// specific method ID. If a decompressor for a given method is not found, +// Reader will default to looking up the decompressor at the package level. +func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) { + if z.decompressors == nil { + z.decompressors = make(map[uint16]Decompressor) + } + z.decompressors[method] = dcomp +} + +func (z *Reader) decompressor(method uint16) Decompressor { + dcomp := z.decompressors[method] + if dcomp == nil { + dcomp = decompressor(method) + } + return dcomp +} + +// Close closes the Zip file, rendering it unusable for I/O. +func (rc *ReadCloser) Close() error { + return rc.f.Close() +} + +// DataOffset returns the offset of the file's possibly-compressed +// data, relative to the beginning of the zip file. +// +// Most callers should instead use Open, which transparently +// decompresses data and verifies checksums. +func (f *File) DataOffset() (offset int64, err error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return + } + return f.headerOffset + bodyOffset, nil +} + +// Open returns a ReadCloser that provides access to the File's contents. +// Multiple files may be read concurrently. +func (f *File) Open() (io.ReadCloser, error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return nil, err + } + size := int64(f.CompressedSize64) + r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size) + dcomp := f.zip.decompressor(f.Method) + if dcomp == nil { + return nil, ErrAlgorithm + } + var rc io.ReadCloser = dcomp(r) + var desr io.Reader + if f.hasDataDescriptor() { + desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen) + } + rc = &checksumReader{ + rc: rc, + hash: crc32.NewIEEE(), + f: f, + desr: desr, + } + return rc, nil +} + +// OpenRaw returns a Reader that returns the *compressed* output of the file. +func (f *File) OpenRaw() (io.Reader, error) { + bodyOffset, err := f.findBodyOffset() + if err != nil { + return nil, err + } + size := int64(f.CompressedSize64) + return io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size), nil +} + +type checksumReader struct { + rc io.ReadCloser + hash hash.Hash32 + nread uint64 // number of bytes read so far + f *File + desr io.Reader // if non-nil, where to read the data descriptor + err error // sticky error +} + +func (r *checksumReader) Read(b []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + n, err = r.rc.Read(b) + r.hash.Write(b[:n]) + r.nread += uint64(n) + if err == nil { + return + } + if err == io.EOF { + if r.nread != r.f.UncompressedSize64 { + return 0, io.ErrUnexpectedEOF + } + if r.desr != nil { + if err1 := readDataDescriptor(r.desr, r.f); err1 != nil { + if err1 == io.EOF { + err = io.ErrUnexpectedEOF + } else { + err = err1 + } + } else if r.hash.Sum32() != r.f.CRC32 { + err = ErrChecksum + } + } else { + // If there's not a data descriptor, we still compare + // the CRC32 of what we've read against the file header + // or TOC's CRC32, if it seems like it was set. + if r.f.CRC32 != 0 && r.hash.Sum32() != r.f.CRC32 { + err = ErrChecksum + } + } + } + r.err = err + return +} + +func (r *checksumReader) Close() error { return r.rc.Close() } + +// findBodyOffset does the minimum work to verify the file has a header +// and returns the file body offset. +func (f *File) findBodyOffset() (int64, error) { + var buf [fileHeaderLen]byte + if _, err := f.zipr.ReadAt(buf[:], f.headerOffset); err != nil { + return 0, err + } + b := readBuf(buf[:]) + if sig := b.uint32(); sig != fileHeaderSignature { + return 0, ErrFormat + } + b = b[22:] // skip over most of the header + filenameLen := int(b.uint16()) + extraLen := int(b.uint16()) + return int64(fileHeaderLen + filenameLen + extraLen), nil +} + +// readDirectoryHeader attempts to read a directory header from r. +// It returns io.ErrUnexpectedEOF if it cannot read a complete header, +// and ErrFormat if it doesn't find a valid header signature. +func readDirectoryHeader(f *File, r io.Reader) error { + var buf [directoryHeaderLen]byte + if _, err := io.ReadFull(r, buf[:]); err != nil { + return err + } + b := readBuf(buf[:]) + if sig := b.uint32(); sig != directoryHeaderSignature { + return ErrFormat + } + f.CreatorVersion = b.uint16() + f.ReaderVersion = b.uint16() + f.Flags = b.uint16() + f.Method = b.uint16() + f.ModifiedTime = b.uint16() + f.ModifiedDate = b.uint16() + f.CRC32 = b.uint32() + f.CompressedSize = b.uint32() + f.UncompressedSize = b.uint32() + f.CompressedSize64 = uint64(f.CompressedSize) + f.UncompressedSize64 = uint64(f.UncompressedSize) + filenameLen := int(b.uint16()) + extraLen := int(b.uint16()) + commentLen := int(b.uint16()) + b = b[4:] // skipped start disk number and internal attributes (2x uint16) + f.ExternalAttrs = b.uint32() + f.headerOffset = int64(b.uint32()) + d := make([]byte, filenameLen+extraLen+commentLen) + if _, err := io.ReadFull(r, d); err != nil { + return err + } + f.Name = string(d[:filenameLen]) + f.Extra = d[filenameLen : filenameLen+extraLen] + f.Comment = string(d[filenameLen+extraLen:]) + + // Determine the character encoding. + utf8Valid1, utf8Require1 := detectUTF8(f.Name) + utf8Valid2, utf8Require2 := detectUTF8(f.Comment) + switch { + case !utf8Valid1 || !utf8Valid2: + // Name and Comment definitely not UTF-8. + f.NonUTF8 = true + case !utf8Require1 && !utf8Require2: + // Name and Comment use only single-byte runes that overlap with UTF-8. + f.NonUTF8 = false + default: + // Might be UTF-8, might be some other encoding; preserve existing flag. + // Some ZIP writers use UTF-8 encoding without setting the UTF-8 flag. + // Since it is impossible to always distinguish valid UTF-8 from some + // other encoding (e.g., GBK or Shift-JIS), we trust the flag. + f.NonUTF8 = f.Flags&0x800 == 0 + } + + needUSize := f.UncompressedSize == ^uint32(0) + needCSize := f.CompressedSize == ^uint32(0) + needHeaderOffset := f.headerOffset == int64(^uint32(0)) + + // Best effort to find what we need. + // Other zip authors might not even follow the basic format, + // and we'll just ignore the Extra content in that case. + var modified time.Time +parseExtras: + for extra := readBuf(f.Extra); len(extra) >= 4; { // need at least tag and size + fieldTag := extra.uint16() + fieldSize := int(extra.uint16()) + if len(extra) < fieldSize { + break + } + fieldBuf := extra.sub(fieldSize) + + switch fieldTag { + case zip64ExtraID: + // update directory values from the zip64 extra block. + // They should only be consulted if the sizes read earlier + // are maxed out. + // See golang.org/issue/13367. + if needUSize { + needUSize = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.UncompressedSize64 = fieldBuf.uint64() + } + if needCSize { + needCSize = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.CompressedSize64 = fieldBuf.uint64() + } + if needHeaderOffset { + needHeaderOffset = false + if len(fieldBuf) < 8 { + return ErrFormat + } + f.headerOffset = int64(fieldBuf.uint64()) + } + case ntfsExtraID: + if len(fieldBuf) < 4 { + continue parseExtras + } + fieldBuf.uint32() // reserved (ignored) + for len(fieldBuf) >= 4 { // need at least tag and size + attrTag := fieldBuf.uint16() + attrSize := int(fieldBuf.uint16()) + if len(fieldBuf) < attrSize { + continue parseExtras + } + attrBuf := fieldBuf.sub(attrSize) + if attrTag != 1 || attrSize != 24 { + continue // Ignore irrelevant attributes + } + + const ticksPerSecond = 1e7 // Windows timestamp resolution + ts := int64(attrBuf.uint64()) // ModTime since Windows epoch + secs := int64(ts / ticksPerSecond) + nsecs := (1e9 / ticksPerSecond) * int64(ts%ticksPerSecond) + epoch := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) + modified = time.Unix(epoch.Unix()+secs, nsecs) + } + case unixExtraID, infoZipUnixExtraID: + if len(fieldBuf) < 8 { + continue parseExtras + } + fieldBuf.uint32() // AcTime (ignored) + ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch + modified = time.Unix(ts, 0) + case extTimeExtraID: + if len(fieldBuf) < 5 || fieldBuf.uint8()&1 == 0 { + continue parseExtras + } + ts := int64(fieldBuf.uint32()) // ModTime since Unix epoch + modified = time.Unix(ts, 0) + } + } + + msdosModified := msDosTimeToTime(f.ModifiedDate, f.ModifiedTime) + f.Modified = msdosModified + if !modified.IsZero() { + f.Modified = modified.UTC() + + // If legacy MS-DOS timestamps are set, we can use the delta between + // the legacy and extended versions to estimate timezone offset. + // + // A non-UTC timezone is always used (even if offset is zero). + // Thus, FileHeader.Modified.Location() == time.UTC is useful for + // determining whether extended timestamps are present. + // This is necessary for users that need to do additional time + // calculations when dealing with legacy ZIP formats. + if f.ModifiedTime != 0 || f.ModifiedDate != 0 { + f.Modified = modified.In(timeZone(msdosModified.Sub(modified))) + } + } + + // Assume that uncompressed size 2³²-1 could plausibly happen in + // an old zip32 file that was sharding inputs into the largest chunks + // possible (or is just malicious; search the web for 42.zip). + // If needUSize is true still, it means we didn't see a zip64 extension. + // As long as the compressed size is not also 2³²-1 (implausible) + // and the header is not also 2³²-1 (equally implausible), + // accept the uncompressed size 2³²-1 as valid. + // If nothing else, this keeps archive/zip working with 42.zip. + _ = needUSize + + if needCSize || needHeaderOffset { + return ErrFormat + } + + return nil +} + +func readDataDescriptor(r io.Reader, f *File) error { + var buf [dataDescriptorLen]byte + + // The spec says: "Although not originally assigned a + // signature, the value 0x08074b50 has commonly been adopted + // as a signature value for the data descriptor record. + // Implementers should be aware that ZIP files may be + // encountered with or without this signature marking data + // descriptors and should account for either case when reading + // ZIP files to ensure compatibility." + // + // dataDescriptorLen includes the size of the signature but + // first read just those 4 bytes to see if it exists. + if _, err := io.ReadFull(r, buf[:4]); err != nil { + return err + } + off := 0 + maybeSig := readBuf(buf[:4]) + if maybeSig.uint32() != dataDescriptorSignature { + // No data descriptor signature. Keep these four + // bytes. + off += 4 + } + if _, err := io.ReadFull(r, buf[off:12]); err != nil { + return err + } + b := readBuf(buf[:12]) + if b.uint32() != f.CRC32 { + return ErrChecksum + } + + // The two sizes that follow here can be either 32 bits or 64 bits + // but the spec is not very clear on this and different + // interpretations has been made causing incompatibilities. We + // already have the sizes from the central directory so we can + // just ignore these. + + return nil +} + +func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) { + // look for directoryEndSignature in the last 1k, then in the last 65k + var buf []byte + var directoryEndOffset int64 + for i, bLen := range []int64{1024, 65 * 1024} { + if bLen > size { + bLen = size + } + buf = make([]byte, int(bLen)) + if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF { + return nil, err + } + if p := findSignatureInBlock(buf); p >= 0 { + buf = buf[p:] + directoryEndOffset = size - bLen + int64(p) + break + } + if i == 1 || bLen == size { + return nil, ErrFormat + } + } + + // read header into struct + b := readBuf(buf[4:]) // skip signature + d := &directoryEnd{ + diskNbr: uint32(b.uint16()), + dirDiskNbr: uint32(b.uint16()), + dirRecordsThisDisk: uint64(b.uint16()), + directoryRecords: uint64(b.uint16()), + directorySize: uint64(b.uint32()), + directoryOffset: uint64(b.uint32()), + commentLen: b.uint16(), + } + l := int(d.commentLen) + if l > len(b) { + return nil, errors.New("zip: invalid comment length") + } + d.comment = string(b[:l]) + + // These values mean that the file can be a zip64 file + if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff { + p, err := findDirectory64End(r, directoryEndOffset) + if err == nil && p >= 0 { + err = readDirectory64End(r, p, d) + } + if err != nil { + return nil, err + } + } + // Make sure directoryOffset points to somewhere in our file. + if o := int64(d.directoryOffset); o < 0 || o >= size { + return nil, ErrFormat + } + return d, nil +} + +// findDirectory64End tries to read the zip64 locator just before the +// directory end and returns the offset of the zip64 directory end if +// found. +func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) { + locOffset := directoryEndOffset - directory64LocLen + if locOffset < 0 { + return -1, nil // no need to look for a header outside the file + } + buf := make([]byte, directory64LocLen) + if _, err := r.ReadAt(buf, locOffset); err != nil { + return -1, err + } + b := readBuf(buf) + if sig := b.uint32(); sig != directory64LocSignature { + return -1, nil + } + if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory + return -1, nil // the file is not a valid zip64-file + } + p := b.uint64() // relative offset of the zip64 end of central directory record + if b.uint32() != 1 { // total number of disks + return -1, nil // the file is not a valid zip64-file + } + return int64(p), nil +} + +// readDirectory64End reads the zip64 directory end and updates the +// directory end with the zip64 directory end values. +func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) { + buf := make([]byte, directory64EndLen) + if _, err := r.ReadAt(buf, offset); err != nil { + return err + } + + b := readBuf(buf) + if sig := b.uint32(); sig != directory64EndSignature { + return ErrFormat + } + + b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16) + d.diskNbr = b.uint32() // number of this disk + d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory + d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk + d.directoryRecords = b.uint64() // total number of entries in the central directory + d.directorySize = b.uint64() // size of the central directory + d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number + + return nil +} + +func findSignatureInBlock(b []byte) int { + for i := len(b) - directoryEndLen; i >= 0; i-- { + // defined from directoryEndSignature in struct.go + if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { + // n is length of comment + n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8 + if n+directoryEndLen+i <= len(b) { + return i + } + } + } + return -1 +} + +type readBuf []byte + +func (b *readBuf) uint8() uint8 { + v := (*b)[0] + *b = (*b)[1:] + return v +} + +func (b *readBuf) uint16() uint16 { + v := binary.LittleEndian.Uint16(*b) + *b = (*b)[2:] + return v +} + +func (b *readBuf) uint32() uint32 { + v := binary.LittleEndian.Uint32(*b) + *b = (*b)[4:] + return v +} + +func (b *readBuf) uint64() uint64 { + v := binary.LittleEndian.Uint64(*b) + *b = (*b)[8:] + return v +} + +func (b *readBuf) sub(n int) readBuf { + b2 := (*b)[:n] + *b = (*b)[n:] + return b2 +} diff --git a/vendor/github.com/klauspost/compress/zip/register.go b/vendor/github.com/klauspost/compress/zip/register.go new file mode 100644 index 0000000000000..3bcb1ffd7ac05 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/register.go @@ -0,0 +1,149 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "errors" + "io" + "io/ioutil" + "sync" + + "github.com/klauspost/compress/flate" +) + +// A Compressor returns a new compressing writer, writing to w. +// The WriteCloser's Close method must be used to flush pending data to w. +// The Compressor itself must be safe to invoke from multiple goroutines +// simultaneously, but each returned writer will be used only by +// one goroutine at a time. +type Compressor func(w io.Writer) (io.WriteCloser, error) + +// A Decompressor returns a new decompressing reader, reading from r. +// The ReadCloser's Close method must be used to release associated resources. +// The Decompressor itself must be safe to invoke from multiple goroutines +// simultaneously, but each returned reader will be used only by +// one goroutine at a time. +type Decompressor func(r io.Reader) io.ReadCloser + +var flateWriterPool sync.Pool + +func newFlateWriter(w io.Writer) io.WriteCloser { + fw, ok := flateWriterPool.Get().(*flate.Writer) + if ok { + fw.Reset(w) + } else { + fw, _ = flate.NewWriter(w, 5) + } + return &pooledFlateWriter{fw: fw} +} + +type pooledFlateWriter struct { + mu sync.Mutex // guards Close and Write + fw *flate.Writer +} + +func (w *pooledFlateWriter) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.fw == nil { + return 0, errors.New("Write after Close") + } + return w.fw.Write(p) +} + +func (w *pooledFlateWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + var err error + if w.fw != nil { + err = w.fw.Close() + flateWriterPool.Put(w.fw) + w.fw = nil + } + return err +} + +var flateReaderPool sync.Pool + +func newFlateReader(r io.Reader) io.ReadCloser { + fr, ok := flateReaderPool.Get().(io.ReadCloser) + if ok { + fr.(flate.Resetter).Reset(r, nil) + } else { + fr = flate.NewReader(r) + } + return &pooledFlateReader{fr: fr} +} + +type pooledFlateReader struct { + mu sync.Mutex // guards Close and Read + fr io.ReadCloser +} + +func (r *pooledFlateReader) Read(p []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.fr == nil { + return 0, errors.New("Read after Close") + } + return r.fr.Read(p) +} + +func (r *pooledFlateReader) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + var err error + if r.fr != nil { + err = r.fr.Close() + flateReaderPool.Put(r.fr) + r.fr = nil + } + return err +} + +var ( + compressors sync.Map // map[uint16]Compressor + decompressors sync.Map // map[uint16]Decompressor +) + +func init() { + compressors.Store(Store, Compressor(func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil })) + compressors.Store(Deflate, Compressor(func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil })) + + decompressors.Store(Store, Decompressor(ioutil.NopCloser)) + decompressors.Store(Deflate, Decompressor(newFlateReader)) +} + +// RegisterDecompressor allows custom decompressors for a specified method ID. +// The common methods Store and Deflate are built in. +func RegisterDecompressor(method uint16, dcomp Decompressor) { + if _, dup := decompressors.LoadOrStore(method, dcomp); dup { + panic("decompressor already registered") + } +} + +// RegisterCompressor registers custom compressors for a specified method ID. +// The common methods Store and Deflate are built in. +func RegisterCompressor(method uint16, comp Compressor) { + if _, dup := compressors.LoadOrStore(method, comp); dup { + panic("compressor already registered") + } +} + +func compressor(method uint16) Compressor { + ci, ok := compressors.Load(method) + if !ok { + return nil + } + return ci.(Compressor) +} + +func decompressor(method uint16) Decompressor { + di, ok := decompressors.Load(method) + if !ok { + return nil + } + return di.(Decompressor) +} diff --git a/vendor/github.com/klauspost/compress/zip/struct.go b/vendor/github.com/klauspost/compress/zip/struct.go new file mode 100644 index 0000000000000..686e79781acb6 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/struct.go @@ -0,0 +1,387 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package zip provides support for reading and writing ZIP archives. + +See: https://www.pkware.com/appnote + +This package does not support disk spanning. + +A note about ZIP64: + +To be backwards compatible the FileHeader has both 32 and 64 bit Size +fields. The 64 bit fields will always contain the correct value and +for normal archives both fields will be the same. For files requiring +the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit +fields must be used instead. +*/ +package zip + +import ( + "os" + "path" + "time" +) + +// Compression methods. +const ( + Store uint16 = 0 // no compression + Deflate uint16 = 8 // DEFLATE compressed +) + +const ( + fileHeaderSignature = 0x04034b50 + directoryHeaderSignature = 0x02014b50 + directoryEndSignature = 0x06054b50 + directory64LocSignature = 0x07064b50 + directory64EndSignature = 0x06064b50 + dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder + fileHeaderLen = 30 // + filename + extra + directoryHeaderLen = 46 // + filename + extra + comment + directoryEndLen = 22 // + comment + dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size + dataDescriptor64Len = 24 // descriptor with 8 byte sizes + directory64LocLen = 20 // + directory64EndLen = 56 // + extra + + // Constants for the first byte in CreatorVersion. + creatorFAT = 0 + creatorUnix = 3 + creatorNTFS = 11 + creatorVFAT = 14 + creatorMacOSX = 19 + + // Version numbers. + zipVersion20 = 20 // 2.0 + zipVersion45 = 45 // 4.5 (reads and writes zip64 archives) + + // Limits for non zip64 files. + uint16max = (1 << 16) - 1 + uint32max = (1 << 32) - 1 + + // Extra header IDs. + // + // IDs 0..31 are reserved for official use by PKWARE. + // IDs above that range are defined by third-party vendors. + // Since ZIP lacked high precision timestamps (nor a official specification + // of the timezone used for the date fields), many competing extra fields + // have been invented. Pervasive use effectively makes them "official". + // + // See http://mdfs.net/Docs/Comp/Archiving/Zip/ExtraField + zip64ExtraID = 0x0001 // Zip64 extended information + ntfsExtraID = 0x000a // NTFS + unixExtraID = 0x000d // UNIX + extTimeExtraID = 0x5455 // Extended timestamp + infoZipUnixExtraID = 0x5855 // Info-ZIP Unix extension +) + +// FileHeader describes a file within a zip file. +// See the zip spec for details. +type FileHeader struct { + // Name is the name of the file. + // + // It must be a relative path, not start with a drive letter (such as "C:"), + // and must use forward slashes instead of back slashes. A trailing slash + // indicates that this file is a directory and should have no data. + // + // When reading zip files, the Name field is populated from + // the zip file directly and is not validated for correctness. + // It is the caller's responsibility to sanitize it as + // appropriate, including canonicalizing slash directions, + // validating that paths are relative, and preventing path + // traversal through filenames ("../../../"). + Name string + + // Comment is any arbitrary user-defined string shorter than 64KiB. + Comment string + + // NonUTF8 indicates that Name and Comment are not encoded in UTF-8. + // + // By specification, the only other encoding permitted should be CP-437, + // but historically many ZIP readers interpret Name and Comment as whatever + // the system's local character encoding happens to be. + // + // This flag should only be set if the user intends to encode a non-portable + // ZIP file for a specific localized region. Otherwise, the Writer + // automatically sets the ZIP format's UTF-8 flag for valid UTF-8 strings. + NonUTF8 bool + + CreatorVersion uint16 + ReaderVersion uint16 + Flags uint16 + + // Method is the compression method. If zero, Store is used. + Method uint16 + + // Modified is the modified time of the file. + // + // When reading, an extended timestamp is preferred over the legacy MS-DOS + // date field, and the offset between the times is used as the timezone. + // If only the MS-DOS date is present, the timezone is assumed to be UTC. + // + // When writing, an extended timestamp (which is timezone-agnostic) is + // always emitted. The legacy MS-DOS date field is encoded according to the + // location of the Modified time. + Modified time.Time + ModifiedTime uint16 // Deprecated: Legacy MS-DOS date; use Modified instead. + ModifiedDate uint16 // Deprecated: Legacy MS-DOS time; use Modified instead. + + CRC32 uint32 + CompressedSize uint32 // Deprecated: Use CompressedSize64 instead. + UncompressedSize uint32 // Deprecated: Use UncompressedSize64 instead. + CompressedSize64 uint64 + UncompressedSize64 uint64 + Extra []byte + ExternalAttrs uint32 // Meaning depends on CreatorVersion +} + +// FileInfo returns an os.FileInfo for the FileHeader. +func (h *FileHeader) FileInfo() os.FileInfo { + return headerFileInfo{h} +} + +// headerFileInfo implements os.FileInfo. +type headerFileInfo struct { + fh *FileHeader +} + +func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) } +func (fi headerFileInfo) Size() int64 { + if fi.fh.UncompressedSize64 > 0 { + return int64(fi.fh.UncompressedSize64) + } + return int64(fi.fh.UncompressedSize) +} +func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi headerFileInfo) ModTime() time.Time { + if fi.fh.Modified.IsZero() { + return fi.fh.ModTime() + } + return fi.fh.Modified.UTC() +} +func (fi headerFileInfo) Mode() os.FileMode { return fi.fh.Mode() } +func (fi headerFileInfo) Sys() interface{} { return fi.fh } + +// FileInfoHeader creates a partially-populated FileHeader from an +// os.FileInfo. +// Because os.FileInfo's Name method returns only the base name of +// the file it describes, it may be necessary to modify the Name field +// of the returned header to provide the full path name of the file. +// If compression is desired, callers should set the FileHeader.Method +// field; it is unset by default. +func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) { + size := fi.Size() + fh := &FileHeader{ + Name: fi.Name(), + UncompressedSize64: uint64(size), + } + fh.SetModTime(fi.ModTime()) + fh.SetMode(fi.Mode()) + if fh.UncompressedSize64 > uint32max { + fh.UncompressedSize = uint32max + } else { + fh.UncompressedSize = uint32(fh.UncompressedSize64) + } + return fh, nil +} + +type directoryEnd struct { + diskNbr uint32 // unused + dirDiskNbr uint32 // unused + dirRecordsThisDisk uint64 // unused + directoryRecords uint64 + directorySize uint64 + directoryOffset uint64 // relative to file + commentLen uint16 + comment string +} + +// timeZone returns a *time.Location based on the provided offset. +// If the offset is non-sensible, then this uses an offset of zero. +func timeZone(offset time.Duration) *time.Location { + const ( + minOffset = -12 * time.Hour // E.g., Baker island at -12:00 + maxOffset = +14 * time.Hour // E.g., Line island at +14:00 + offsetAlias = 15 * time.Minute // E.g., Nepal at +5:45 + ) + offset = offset.Round(offsetAlias) + if offset < minOffset || maxOffset < offset { + offset = 0 + } + return time.FixedZone("", int(offset/time.Second)) +} + +// msDosTimeToTime converts an MS-DOS date and time into a time.Time. +// The resolution is 2s. +// See: https://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx +func msDosTimeToTime(dosDate, dosTime uint16) time.Time { + return time.Date( + // date bits 0-4: day of month; 5-8: month; 9-15: years since 1980 + int(dosDate>>9+1980), + time.Month(dosDate>>5&0xf), + int(dosDate&0x1f), + + // time bits 0-4: second/2; 5-10: minute; 11-15: hour + int(dosTime>>11), + int(dosTime>>5&0x3f), + int(dosTime&0x1f*2), + 0, // nanoseconds + + time.UTC, + ) +} + +// timeToMsDosTime converts a time.Time to an MS-DOS date and time. +// The resolution is 2s. +// See: https://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx +func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { + fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9) + fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11) + return +} + +// ModTime returns the modification time in UTC using the legacy +// ModifiedDate and ModifiedTime fields. +// +// Deprecated: Use Modified instead. +func (h *FileHeader) ModTime() time.Time { + return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) +} + +// SetModTime sets the Modified, ModifiedTime, and ModifiedDate fields +// to the given time in UTC. +// +// Deprecated: Use Modified instead. +func (h *FileHeader) SetModTime(t time.Time) { + t = t.UTC() // Convert to UTC for compatibility + h.Modified = t + h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t) +} + +const ( + // Unix constants. The specification doesn't mention them, + // but these seem to be the values agreed on by tools. + s_IFMT = 0xf000 + s_IFSOCK = 0xc000 + s_IFLNK = 0xa000 + s_IFREG = 0x8000 + s_IFBLK = 0x6000 + s_IFDIR = 0x4000 + s_IFCHR = 0x2000 + s_IFIFO = 0x1000 + s_ISUID = 0x800 + s_ISGID = 0x400 + s_ISVTX = 0x200 + + msdosDir = 0x10 + msdosReadOnly = 0x01 +) + +// Mode returns the permission and mode bits for the FileHeader. +func (h *FileHeader) Mode() (mode os.FileMode) { + switch h.CreatorVersion >> 8 { + case creatorUnix, creatorMacOSX: + mode = unixModeToFileMode(h.ExternalAttrs >> 16) + case creatorNTFS, creatorVFAT, creatorFAT: + mode = msdosModeToFileMode(h.ExternalAttrs) + } + if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' { + mode |= os.ModeDir + } + return mode +} + +// SetMode changes the permission and mode bits for the FileHeader. +func (h *FileHeader) SetMode(mode os.FileMode) { + h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8 + h.ExternalAttrs = fileModeToUnixMode(mode) << 16 + + // set MSDOS attributes too, as the original zip does. + if mode&os.ModeDir != 0 { + h.ExternalAttrs |= msdosDir + } + if mode&0200 == 0 { + h.ExternalAttrs |= msdosReadOnly + } +} + +// isZip64 reports whether the file size exceeds the 32 bit limit +func (h *FileHeader) isZip64() bool { + return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max +} + +func msdosModeToFileMode(m uint32) (mode os.FileMode) { + if m&msdosDir != 0 { + mode = os.ModeDir | 0777 + } else { + mode = 0666 + } + if m&msdosReadOnly != 0 { + mode &^= 0222 + } + return mode +} + +func fileModeToUnixMode(mode os.FileMode) uint32 { + var m uint32 + switch mode & os.ModeType { + default: + m = s_IFREG + case os.ModeDir: + m = s_IFDIR + case os.ModeSymlink: + m = s_IFLNK + case os.ModeNamedPipe: + m = s_IFIFO + case os.ModeSocket: + m = s_IFSOCK + case os.ModeDevice: + if mode&os.ModeCharDevice != 0 { + m = s_IFCHR + } else { + m = s_IFBLK + } + } + if mode&os.ModeSetuid != 0 { + m |= s_ISUID + } + if mode&os.ModeSetgid != 0 { + m |= s_ISGID + } + if mode&os.ModeSticky != 0 { + m |= s_ISVTX + } + return m | uint32(mode&0777) +} + +func unixModeToFileMode(m uint32) os.FileMode { + mode := os.FileMode(m & 0777) + switch m & s_IFMT { + case s_IFBLK: + mode |= os.ModeDevice + case s_IFCHR: + mode |= os.ModeDevice | os.ModeCharDevice + case s_IFDIR: + mode |= os.ModeDir + case s_IFIFO: + mode |= os.ModeNamedPipe + case s_IFLNK: + mode |= os.ModeSymlink + case s_IFREG: + // nothing to do + case s_IFSOCK: + mode |= os.ModeSocket + } + if m&s_ISGID != 0 { + mode |= os.ModeSetgid + } + if m&s_ISUID != 0 { + mode |= os.ModeSetuid + } + if m&s_ISVTX != 0 { + mode |= os.ModeSticky + } + return mode +} diff --git a/vendor/github.com/klauspost/compress/zip/writer.go b/vendor/github.com/klauspost/compress/zip/writer.go new file mode 100644 index 0000000000000..335b637c8f69b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zip/writer.go @@ -0,0 +1,747 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zip + +import ( + "bufio" + "encoding/binary" + "errors" + "hash" + "hash/crc32" + "io" + "strings" + "unicode/utf8" +) + +var ( + errLongName = errors.New("zip: FileHeader.Name too long") + errLongExtra = errors.New("zip: FileHeader.Extra too long") +) + +type lastWriter interface { + Close() error + Closed() bool +} + +// Writer implements a zip file writer. +type Writer struct { + cw *countWriter + dir []*header + last lastWriter + closed bool + compressors map[uint16]Compressor + comment string + + // testHookCloseSizeOffset if non-nil is called with the size + // of offset of the central directory at Close. + testHookCloseSizeOffset func(size, offset uint64) +} + +type header struct { + *FileHeader + offset uint64 +} + +// NewWriter returns a new Writer writing a zip file to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}} +} + +// SetOffset sets the offset of the beginning of the zip data within the +// underlying writer. It should be used when the zip data is appended to an +// existing file, such as a binary executable. +// It must be called before any data is written. +func (w *Writer) SetOffset(n int64) { + if w.cw.count != 0 { + panic("zip: SetOffset called after data was written") + } + w.cw.count = n +} + +// Flush flushes any buffered data to the underlying writer. +// Calling Flush is not normally necessary; calling Close is sufficient. +func (w *Writer) Flush() error { + return w.cw.w.(*bufio.Writer).Flush() +} + +// SetComment sets the end-of-central-directory comment field. +// It can only be called before Close. +func (w *Writer) SetComment(comment string) error { + if len(comment) > uint16max { + return errors.New("zip: Writer.Comment too long") + } + w.comment = comment + return nil +} + +// Close finishes writing the zip file by writing the central directory. +// It does not Close the underlying writer. +func (w *Writer) Close() error { + if w.last != nil && !w.last.Closed() { + if err := w.last.Close(); err != nil { + return err + } + w.last = nil + } + if w.closed { + return errors.New("zip: writer closed twice") + } + w.closed = true + + // write central directory + start := w.cw.count + for _, h := range w.dir { + var buf [directoryHeaderLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(directoryHeaderSignature)) + b.uint16(h.CreatorVersion) + b.uint16(h.ReaderVersion) + b.uint16(h.Flags) + b.uint16(h.Method) + b.uint16(h.ModifiedTime) + b.uint16(h.ModifiedDate) + b.uint32(h.CRC32) + if h.isZip64() || h.offset >= uint32max { + // the file needs a zip64 header. store maxint in both + // 32 bit size fields (and offset later) to signal that the + // zip64 extra header should be used. + b.uint32(uint32max) // compressed size + b.uint32(uint32max) // uncompressed size + + // append a zip64 extra block to Extra + var buf [28]byte // 2x uint16 + 3x uint64 + eb := writeBuf(buf[:]) + eb.uint16(zip64ExtraID) + eb.uint16(24) // size = 3x uint64 + eb.uint64(h.UncompressedSize64) + eb.uint64(h.CompressedSize64) + eb.uint64(h.offset) + h.Extra = append(h.Extra, buf[:]...) + } else { + b.uint32(h.CompressedSize) + b.uint32(h.UncompressedSize) + } + + b.uint16(uint16(len(h.Name))) + b.uint16(uint16(len(h.Extra))) + b.uint16(uint16(len(h.Comment))) + b = b[4:] // skip disk number start and internal file attr (2x uint16) + b.uint32(h.ExternalAttrs) + if h.offset > uint32max { + b.uint32(uint32max) + } else { + b.uint32(uint32(h.offset)) + } + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w.cw, h.Name); err != nil { + return err + } + if _, err := w.cw.Write(h.Extra); err != nil { + return err + } + if _, err := io.WriteString(w.cw, h.Comment); err != nil { + return err + } + } + end := w.cw.count + + records := uint64(len(w.dir)) + size := uint64(end - start) + offset := uint64(start) + + if f := w.testHookCloseSizeOffset; f != nil { + f(size, offset) + } + + if records >= uint16max || size >= uint32max || offset >= uint32max { + var buf [directory64EndLen + directory64LocLen]byte + b := writeBuf(buf[:]) + + // zip64 end of central directory record + b.uint32(directory64EndSignature) + b.uint64(directory64EndLen - 12) // length minus signature (uint32) and length fields (uint64) + b.uint16(zipVersion45) // version made by + b.uint16(zipVersion45) // version needed to extract + b.uint32(0) // number of this disk + b.uint32(0) // number of the disk with the start of the central directory + b.uint64(records) // total number of entries in the central directory on this disk + b.uint64(records) // total number of entries in the central directory + b.uint64(size) // size of the central directory + b.uint64(offset) // offset of start of central directory with respect to the starting disk number + + // zip64 end of central directory locator + b.uint32(directory64LocSignature) + b.uint32(0) // number of the disk with the start of the zip64 end of central directory + b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record + b.uint32(1) // total number of disks + + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + + // store max values in the regular end record to signal + // that the zip64 values should be used instead + records = uint16max + size = uint32max + offset = uint32max + } + + // write end record + var buf [directoryEndLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(directoryEndSignature)) + b = b[4:] // skip over disk number and first disk number (2x uint16) + b.uint16(uint16(records)) // number of entries this disk + b.uint16(uint16(records)) // number of entries total + b.uint32(uint32(size)) // size of directory + b.uint32(uint32(offset)) // start of directory + b.uint16(uint16(len(w.comment))) // byte size of EOCD comment + if _, err := w.cw.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w.cw, w.comment); err != nil { + return err + } + + return w.cw.w.(*bufio.Writer).Flush() +} + +// Create adds a file to the zip file using the provided name. +// It returns a Writer to which the file contents should be written. +// The file contents will be compressed using the Deflate method. +// The name must be a relative path: it must not start with a drive +// letter (e.g. C:) or leading slash, and only forward slashes are +// allowed. To create a directory instead of a file, add a trailing +// slash to the name. +// The file's contents must be written to the io.Writer before the next +// call to Create, CreateHeader, CreateHeaderRaw, or Close. +func (w *Writer) Create(name string) (io.Writer, error) { + header := &FileHeader{ + Name: name, + Method: Deflate, + } + return w.CreateHeader(header) +} + +// Copy will copy raw content from input file. +// Optionally a different name can be given to the new file. +func (w *Writer) Copy(name string, src *File) error { + header := src.FileHeader + if name != "" { + header.Name = name + } + raw, err := src.OpenRaw() + if err != nil { + return err + } + wr, err := w.CreateHeaderRaw(&header) + if err != nil { + return err + } + _, err = io.Copy(wr, raw) + return err +} + +// detectUTF8 reports whether s is a valid UTF-8 string, and whether the string +// must be considered UTF-8 encoding (i.e., not compatible with CP-437, ASCII, +// or any other common encoding). +func detectUTF8(s string) (valid, require bool) { + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + i += size + // Officially, ZIP uses CP-437, but many readers use the system's + // local character encoding. Most encoding are compatible with a large + // subset of CP-437, which itself is ASCII-like. + // + // Forbid 0x7e and 0x5c since EUC-KR and Shift-JIS replace those + // characters with localized currency and overline characters. + if r < 0x20 || r > 0x7d || r == 0x5c { + if !utf8.ValidRune(r) || (r == utf8.RuneError && size == 1) { + return false, false + } + require = true + } + } + return true, require +} + +// CreateHeader adds a file to the zip archive using the provided FileHeader +// for the file metadata. Writer takes ownership of fh and may mutate +// its fields. The caller must not modify fh after calling CreateHeader. +// +// This returns a Writer to which the file contents should be written. +// The file's contents must be written to the io.Writer before the next +// call to Create, Copy, CreateHeader, CreateHeaderRaw or Close. +func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) { + if w.last != nil && !w.last.Closed() { + if err := w.last.Close(); err != nil { + return nil, err + } + } + if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh { + // See https://golang.org/issue/11144 confusion. + return nil, errors.New("archive/zip: invalid duplicate FileHeader") + } + + // The ZIP format has a sad state of affairs regarding character encoding. + // Officially, the name and comment fields are supposed to be encoded + // in CP-437 (which is mostly compatible with ASCII), unless the UTF-8 + // flag bit is set. However, there are several problems: + // + // * Many ZIP readers still do not support UTF-8. + // * If the UTF-8 flag is cleared, several readers simply interpret the + // name and comment fields as whatever the local system encoding is. + // + // In order to avoid breaking readers without UTF-8 support, + // we avoid setting the UTF-8 flag if the strings are CP-437 compatible. + // However, if the strings require multibyte UTF-8 encoding and is a + // valid UTF-8 string, then we set the UTF-8 bit. + // + // For the case, where the user explicitly wants to specify the encoding + // as UTF-8, they will need to set the flag bit themselves. + utf8Valid1, utf8Require1 := detectUTF8(fh.Name) + utf8Valid2, utf8Require2 := detectUTF8(fh.Comment) + switch { + case fh.NonUTF8: + fh.Flags &^= 0x800 + case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2): + fh.Flags |= 0x800 + } + + fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte + fh.ReaderVersion = zipVersion20 + + // If Modified is set, this takes precedence over MS-DOS timestamp fields. + if !fh.Modified.IsZero() { + // Contrary to the FileHeader.SetModTime method, we intentionally + // do not convert to UTC, because we assume the user intends to encode + // the date using the specified timezone. A user may want this control + // because many legacy ZIP readers interpret the timestamp according + // to the local timezone. + // + // The timezone is only non-UTC if a user directly sets the Modified + // field directly themselves. All other approaches sets UTC. + fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified) + + // Use "extended timestamp" format since this is what Info-ZIP uses. + // Nearly every major ZIP implementation uses a different format, + // but at least most seem to be able to understand the other formats. + // + // This format happens to be identical for both local and central header + // if modification time is the only timestamp being encoded. + var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32) + mt := uint32(fh.Modified.Unix()) + eb := writeBuf(mbuf[:]) + eb.uint16(extTimeExtraID) + eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32) + eb.uint8(1) // Flags: ModTime + eb.uint32(mt) // ModTime + fh.Extra = append(fh.Extra, mbuf[:]...) + } + + var ow io.Writer + h := &header{ + FileHeader: fh, + offset: uint64(w.cw.count), + } + + if strings.HasSuffix(fh.Name, "/") { + // Set the compression method to Store to ensure data length is truly zero, + // which the writeHeader method always encodes for the size fields. + // This is necessary as most compression formats have non-zero lengths + // even when compressing an empty string. + fh.Method = Store + fh.Flags &^= 0x8 // we will not write a data descriptor + + // Explicitly clear sizes as they have no meaning for directories. + fh.CompressedSize = 0 + fh.CompressedSize64 = 0 + fh.UncompressedSize = 0 + fh.UncompressedSize64 = 0 + + ow = dirWriter{} + w.last = nil + } else { + fh.Flags |= 0x8 // we will write a data descriptor + + fw := &fileWriter{ + zipw: w.cw, + compCount: &countWriter{w: w.cw}, + crc32: crc32.NewIEEE(), + } + comp := w.compressor(fh.Method) + if comp == nil { + return nil, ErrAlgorithm + } + var err error + fw.comp, err = comp(fw.compCount) + if err != nil { + return nil, err + } + fw.rawCount = &countWriter{w: fw.comp} + fw.header = h + ow = fw + w.last = fw + } + w.dir = append(w.dir, h) + if err := writeHeader(w.cw, fh); err != nil { + return nil, err + } + // If we're creating a directory, fw is nil. + return ow, nil +} + +// CreateHeaderRaw adds a file to the zip archive using the provided FileHeader +// for the file metadata. Writer takes ownership of fh and may mutate +// its fields. The caller must not modify fh after calling CreateHeaderRaw. +// +// This returns a Writer to which the compressed file contents should be written. +// The file's contents must be written to the io.Writer before the next +// call to Create, Copy, CreateHeader, CreateHeaderRaw or Close. +// +// Using this requires knowledge of populating the FileHeader correctly (the +// UncompressedSize64 and CRC32 fields should be set and valid for the contents +// written). For copying from an existing zip file, the Copy() function is +// recommended. +func (w *Writer) CreateHeaderRaw(fh *FileHeader) (io.Writer, error) { + if w.last != nil && !w.last.Closed() { + if err := w.last.Close(); err != nil { + return nil, err + } + } + if len(w.dir) > 0 && w.dir[len(w.dir)-1].FileHeader == fh { + // See https://golang.org/issue/11144 confusion. + return nil, errors.New("archive/zip: invalid duplicate FileHeader") + } + + // The ZIP format has a sad state of affairs regarding character encoding. + // Officially, the name and comment fields are supposed to be encoded + // in CP-437 (which is mostly compatible with ASCII), unless the UTF-8 + // flag bit is set. However, there are several problems: + // + // * Many ZIP readers still do not support UTF-8. + // * If the UTF-8 flag is cleared, several readers simply interpret the + // name and comment fields as whatever the local system encoding is. + // + // In order to avoid breaking readers without UTF-8 support, + // we avoid setting the UTF-8 flag if the strings are CP-437 compatible. + // However, if the strings require multibyte UTF-8 encoding and is a + // valid UTF-8 string, then we set the UTF-8 bit. + // + // For the case, where the user explicitly wants to specify the encoding + // as UTF-8, they will need to set the flag bit themselves. + utf8Valid1, utf8Require1 := detectUTF8(fh.Name) + utf8Valid2, utf8Require2 := detectUTF8(fh.Comment) + switch { + case fh.NonUTF8: + fh.Flags &^= 0x800 + case (utf8Require1 || utf8Require2) && (utf8Valid1 && utf8Valid2): + fh.Flags |= 0x800 + } + + fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte + fh.ReaderVersion = zipVersion20 + + // If Modified is set, this takes precedence over MS-DOS timestamp fields. + if !fh.Modified.IsZero() { + // Contrary to the FileHeader.SetModTime method, we intentionally + // do not convert to UTC, because we assume the user intends to encode + // the date using the specified timezone. A user may want this control + // because many legacy ZIP readers interpret the timestamp according + // to the local timezone. + // + // The timezone is only non-UTC if a user directly sets the Modified + // field directly themselves. All other approaches sets UTC. + fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(fh.Modified) + + // Use "extended timestamp" format since this is what Info-ZIP uses. + // Nearly every major ZIP implementation uses a different format, + // but at least most seem to be able to understand the other formats. + // + // This format happens to be identical for both local and central header + // if modification time is the only timestamp being encoded. + var mbuf [9]byte // 2*SizeOf(uint16) + SizeOf(uint8) + SizeOf(uint32) + mt := uint32(fh.Modified.Unix()) + eb := writeBuf(mbuf[:]) + eb.uint16(extTimeExtraID) + eb.uint16(5) // Size: SizeOf(uint8) + SizeOf(uint32) + eb.uint8(1) // Flags: ModTime + eb.uint32(mt) // ModTime + fh.Extra = append(fh.Extra, mbuf[:]...) + } + + var ow io.Writer + h := &header{ + FileHeader: fh, + offset: uint64(w.cw.count), + } + + if strings.HasSuffix(fh.Name, "/") { + // Set the compression method to Store to ensure data length is truly zero, + // which the writeHeader method always encodes for the size fields. + // This is necessary as most compression formats have non-zero lengths + // even when compressing an empty string. + fh.Method = Store + fh.Flags &^= 0x8 // we will not write a data descriptor + + // Explicitly clear sizes as they have no meaning for directories. + fh.CompressedSize = 0 + fh.CompressedSize64 = 0 + fh.UncompressedSize = 0 + fh.UncompressedSize64 = 0 + + ow = dirWriter{} + w.last = nil + } else { + fh.Flags |= 0x8 // we will write a data descriptor + + fw := &rawWriter{ + header: h, + zipw: w.cw, + rawCount: &countWriter{w: w.cw}, + } + ow = fw + w.last = fw + } + w.dir = append(w.dir, h) + if err := writeHeader(w.cw, fh); err != nil { + return nil, err + } + // If we're creating a directory, fw is nil. + return ow, nil +} + +func writeHeader(w io.Writer, h *FileHeader) error { + const maxUint16 = 1<<16 - 1 + if len(h.Name) > maxUint16 { + return errLongName + } + if len(h.Extra) > maxUint16 { + return errLongExtra + } + + var buf [fileHeaderLen]byte + b := writeBuf(buf[:]) + b.uint32(uint32(fileHeaderSignature)) + b.uint16(h.ReaderVersion) + b.uint16(h.Flags) + b.uint16(h.Method) + b.uint16(h.ModifiedTime) + b.uint16(h.ModifiedDate) + b.uint32(0) // since we are writing a data descriptor crc32, + b.uint32(0) // compressed size, + b.uint32(0) // and uncompressed size should be zero + b.uint16(uint16(len(h.Name))) + b.uint16(uint16(len(h.Extra))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w, h.Name); err != nil { + return err + } + _, err := w.Write(h.Extra) + return err +} + +// RegisterCompressor registers or overrides a custom compressor for a specific +// method ID. If a compressor for a given method is not found, Writer will +// default to looking up the compressor at the package level. +func (w *Writer) RegisterCompressor(method uint16, comp Compressor) { + if w.compressors == nil { + w.compressors = make(map[uint16]Compressor) + } + w.compressors[method] = comp +} + +func (w *Writer) compressor(method uint16) Compressor { + comp := w.compressors[method] + if comp == nil { + comp = compressor(method) + } + return comp +} + +type dirWriter struct{} + +func (dirWriter) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + return 0, errors.New("zip: write to directory") +} + +type fileWriter struct { + *header + zipw io.Writer + rawCount *countWriter + comp io.WriteCloser + compCount *countWriter + crc32 hash.Hash32 + closed bool +} + +func (w *fileWriter) Write(p []byte) (int, error) { + if w.closed { + return 0, errors.New("zip: write to closed file") + } + w.crc32.Write(p) + return w.rawCount.Write(p) +} + +func (w *fileWriter) Closed() bool { + return w.closed +} + +func (w *fileWriter) Close() error { + if w.closed { + return errors.New("zip: file closed twice") + } + w.closed = true + if err := w.comp.Close(); err != nil { + return err + } + + // update FileHeader + fh := w.header.FileHeader + fh.CRC32 = w.crc32.Sum32() + fh.CompressedSize64 = uint64(w.compCount.count) + fh.UncompressedSize64 = uint64(w.rawCount.count) + + if fh.isZip64() { + fh.CompressedSize = uint32max + fh.UncompressedSize = uint32max + fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions + } else { + fh.CompressedSize = uint32(fh.CompressedSize64) + fh.UncompressedSize = uint32(fh.UncompressedSize64) + } + + // Write data descriptor. This is more complicated than one would + // think, see e.g. comments in zipfile.c:putextended() and + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588. + // The approach here is to write 8 byte sizes if needed without + // adding a zip64 extra in the local header (too late anyway). + var buf []byte + if fh.isZip64() { + buf = make([]byte, dataDescriptor64Len) + } else { + buf = make([]byte, dataDescriptorLen) + } + b := writeBuf(buf) + b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X + b.uint32(fh.CRC32) + if fh.isZip64() { + b.uint64(fh.CompressedSize64) + b.uint64(fh.UncompressedSize64) + } else { + b.uint32(fh.CompressedSize) + b.uint32(fh.UncompressedSize) + } + _, err := w.zipw.Write(buf) + return err +} + +type rawWriter struct { + *header + zipw io.Writer + rawCount *countWriter + closed bool +} + +func (w *rawWriter) Write(p []byte) (int, error) { + if w.closed { + return 0, errors.New("zip: write to closed file") + } + return w.rawCount.Write(p) +} + +func (w *rawWriter) Closed() bool { + return w.closed +} + +func (w *rawWriter) Close() error { + if w.closed { + return errors.New("zip: file closed twice") + } + w.closed = true + fh := w.FileHeader + fh.CompressedSize64 = uint64(w.rawCount.count) + + if fh.isZip64() { + fh.CompressedSize = uint32max + fh.UncompressedSize = uint32max + fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions + } else { + fh.CompressedSize = uint32(fh.CompressedSize64) + fh.UncompressedSize = uint32(fh.UncompressedSize64) + } + + // Write data descriptor. This is more complicated than one would + // think, see e.g. comments in zipfile.c:putextended() and + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588. + // The approach here is to write 8 byte sizes if needed without + // adding a zip64 extra in the local header (too late anyway). + var buf []byte + if fh.isZip64() { + buf = make([]byte, dataDescriptor64Len) + } else { + buf = make([]byte, dataDescriptorLen) + } + b := writeBuf(buf) + b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X + b.uint32(fh.CRC32) + if fh.isZip64() { + b.uint64(fh.CompressedSize64) + b.uint64(fh.UncompressedSize64) + } else { + b.uint32(fh.CompressedSize) + b.uint32(fh.UncompressedSize) + } + _, err := w.zipw.Write(buf) + return err +} + +type countWriter struct { + w io.Writer + count int64 +} + +func (w *countWriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.count += int64(n) + return n, err +} + +type nopCloser struct { + io.Writer +} + +func (w nopCloser) Close() error { + return nil +} + +type writeBuf []byte + +func (b *writeBuf) uint8(v uint8) { + (*b)[0] = v + *b = (*b)[1:] +} + +func (b *writeBuf) uint16(v uint16) { + binary.LittleEndian.PutUint16(*b, v) + *b = (*b)[2:] +} + +func (b *writeBuf) uint32(v uint32) { + binary.LittleEndian.PutUint32(*b, v) + *b = (*b)[4:] +} + +func (b *writeBuf) uint64(v uint64) { + binary.LittleEndian.PutUint64(*b, v) + *b = (*b)[8:] +} diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index ac3640dc90fe8..7680bfe1dd6c9 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -5,7 +5,6 @@ It offers a very wide range of compression / speed trade-off, while being backed A high performance compression algorithm is implemented. For now focused on speed. This package provides [compression](#Compressor) to and [decompression](#Decompressor) of Zstandard content. -Note that custom dictionaries are only supported for decompression. This package is pure Go and without use of "unsafe". @@ -25,22 +24,21 @@ Godoc Documentation: https://godoc.org/github.com/klauspost/compress/zstd ### Status: STABLE - there may always be subtle bugs, a wide variety of content has been tested and the library is actively -used by several projects. This library is being continuously [fuzz-tested](https://github.com/klauspost/compress-fuzz), -kindly supplied by [fuzzit.dev](https://fuzzit.dev/). +used by several projects. This library is being [fuzz-tested](https://github.com/klauspost/compress-fuzz) for all updates. There may still be specific combinations of data types/size/settings that could lead to edge cases, so as always, testing is recommended. For now, a high speed (fastest) and medium-fast (default) compressor has been implemented. -The "Fastest" compression ratio is roughly equivalent to zstd level 1. -The "Default" compression ratio is roughly equivalent to zstd level 3 (default). +* The "Fastest" compression ratio is roughly equivalent to zstd level 1. +* The "Default" compression ratio is roughly equivalent to zstd level 3 (default). +* The "Better" compression ratio is roughly equivalent to zstd level 7. +* The "Best" compression ratio is roughly equivalent to zstd level 11. In terms of speed, it is typically 2x as fast as the stdlib deflate/gzip in its fastest mode. The compression ratio compared to stdlib is around level 3, but usually 3x as fast. -Compared to cgo zstd, the speed is around level 3 (default), but compression slightly worse, between level 1&2. - ### Usage @@ -55,11 +53,11 @@ To create a writer with default options, do like this: ```Go // Compress input to output. func Compress(in io.Reader, out io.Writer) error { - w, err := NewWriter(output) + enc, err := zstd.NewWriter(out) if err != nil { return err } - _, err := io.Copy(w, input) + _, err = io.Copy(enc, in) if err != nil { enc.Close() return err @@ -141,7 +139,7 @@ I have collected some speed examples to compare speed and compression against ot * `file` is the input file. * `out` is the compressor used. `zskp` is this package. `zstd` is the Datadog cgo library. `gzstd/gzkp` is gzip standard and this library. -* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default". +* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default"; 3 is "better", 4 is "best". * `insize`/`outsize` is the input/output size. * `millis` is the number of milliseconds used for compression. * `mb/s` is megabytes (2^20 bytes) per second. @@ -155,11 +153,13 @@ file out level insize outsize millis mb/s silesia.tar zskp 1 211947520 73101992 643 313.87 silesia.tar zskp 2 211947520 67504318 969 208.38 silesia.tar zskp 3 211947520 65177448 1899 106.44 +silesia.tar zskp 4 211947520 61381950 8115 24.91 cgo zstd: silesia.tar zstd 1 211947520 73605392 543 371.56 silesia.tar zstd 3 211947520 66793289 864 233.68 silesia.tar zstd 6 211947520 62916450 1913 105.66 +silesia.tar zstd 9 211947520 60212393 5063 39.92 gzip, stdlib/this package: silesia.tar gzstd 1 211947520 80007735 1654 122.21 @@ -172,9 +172,11 @@ file out level insize outsize millis mb/s gob-stream zskp 1 1911399616 235022249 3088 590.30 gob-stream zskp 2 1911399616 205669791 3786 481.34 gob-stream zskp 3 1911399616 185792019 9324 195.48 +gob-stream zskp 4 1911399616 171537212 32113 56.76 gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 6 1911399616 193632038 6687 272.56 +gob-stream zstd 9 1911399616 177620386 16175 112.70 gob-stream gzstd 1 1911399616 357382641 10251 177.82 gob-stream gzkp 1 1911399616 362156523 5695 320.08 @@ -186,9 +188,11 @@ file out level insize outsize millis mb/s enwik9 zskp 1 1000000000 343848582 3609 264.18 enwik9 zskp 2 1000000000 317276632 5746 165.97 enwik9 zskp 3 1000000000 294540704 11725 81.34 +enwik9 zskp 4 1000000000 276609671 44029 21.66 enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 6 1000000000 295138875 10290 92.68 +enwik9 zstd 9 1000000000 278348700 28549 33.40 enwik9 gzstd 1 1000000000 382578136 9604 99.30 enwik9 gzkp 1 1000000000 383825945 6544 145.73 @@ -199,9 +203,11 @@ file out level insize outsize millis mb/s github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 github-june-2days-2019.json zskp 3 6273951764 537511906 29252 204.54 +github-june-2days-2019.json zskp 4 6273951764 512796117 97791 61.18 github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 +github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79 github-june-2days-2019.json gzkp 1 6273951764 1128755542 19236 311.03 @@ -212,9 +218,11 @@ file out level insize outsize millis mb/s rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 rawstudio-mint14.tar zskp 3 8558382592 3224594213 71751 113.75 +rawstudio-mint14.tar zskp 4 8558382592 3027332295 486243 16.79 rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 +rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40 rawstudio-mint14.tar gzkp 1 8558382592 3970463184 41749 195.49 @@ -225,48 +233,15 @@ file out level insize outsize millis mb/s nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 nyc-taxi-data-10M.csv zskp 3 3325605752 538490114 19880 159.53 +nyc-taxi-data-10M.csv zskp 4 3325605752 495986829 89368 35.49 nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 +nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83 nyc-taxi-data-10M.csv gzkp 1 3325605752 924718719 16388 193.53 ``` -### Converters - -As part of the development process a *Snappy* -> *Zstandard* converter was also built. - -This can convert a *framed* [Snappy Stream](https://godoc.org/github.com/golang/snappy#Writer) to a zstd stream. -Note that a single block is not framed. - -Conversion is done by converting the stream directly from Snappy without intermediate full decoding. -Therefore the compression ratio is much less than what can be done by a full decompression -and compression, and a faulty Snappy stream may lead to a faulty Zstandard stream without -any errors being generated. -No CRC value is being generated and not all CRC values of the Snappy stream are checked. -However, it provides really fast re-compression of Snappy streams. - - -``` -BenchmarkSnappy_ConvertSilesia-8 1 1156001600 ns/op 183.35 MB/s -Snappy len 103008711 -> zstd len 82687318 - -BenchmarkSnappy_Enwik9-8 1 6472998400 ns/op 154.49 MB/s -Snappy len 508028601 -> zstd len 390921079 -``` - - -```Go - s := zstd.SnappyConverter{} - n, err = s.Convert(input, output) - if err != nil { - fmt.Println("Re-compressed stream to", n, "bytes") - } -``` - -The converter `s` can be reused to avoid allocations, even after errors. - - ## Decompressor Staus: STABLE - there may still be subtle bugs, but a wide variety of content has been tested. @@ -287,14 +262,14 @@ For streaming use a simple setup could look like this: import "github.com/klauspost/compress/zstd" func Decompress(in io.Reader, out io.Writer) error { - d, err := zstd.NewReader(input) + d, err := zstd.NewReader(in) if err != nil { return err } defer d.Close() // Copy content... - _, err := io.Copy(out, d) + _, err = io.Copy(out, d) return err } ``` @@ -337,6 +312,21 @@ A re-used Decoder will still contain the dictionaries registered. When registering multiple dictionaries with the same ID, the last one will be used. +It is possible to use dictionaries when compressing data. + +To enable a dictionary use `WithEncoderDict(dict []byte)`. Here only one dictionary will be used +and it will likely be used even if it doesn't improve compression. + +The used dictionary must be used to decompress the content. + +For any real gains, the dictionary should be built with similar data. +If an unsuitable dictionary is used the output may be slightly larger than using no dictionary. +Use the [zstd commandline tool](https://github.com/facebook/zstd/releases) to build a dictionary from sample data. +For information see [zstd dictionary information](https://github.com/facebook/zstd#the-case-for-small-data-compression). + +For now there is a fixed startup performance penalty for compressing content with dictionaries. +This will likely be improved over time. Just be aware to test performance when implementing. + ### Allocation-less operation The decoder has been designed to operate without allocations after a warmup. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index c8ec6e3312df0..b51d922bda691 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -613,7 +613,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { // Decode treeless literal block. if litType == literalsBlockTreeless { // TODO: We could send the history early WITHOUT the stream history. - // This would allow decoding treeless literials before the byte history is available. + // This would allow decoding treeless literals before the byte history is available. // Silencia stats: Treeless 4393, with: 32775, total: 37168, 11% treeless. // So not much obvious gain here. @@ -646,7 +646,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { } } else { if hist.huffTree != nil && huff != nil { - if hist.dict == nil || hist.dict.litDec != hist.huffTree { + if hist.dict == nil || hist.dict.litEnc != hist.huffTree { huffDecoderPool.Put(hist.huffTree) } hist.huffTree = nil diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index c584f6aabc01d..c85c40255d5f4 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -14,12 +14,13 @@ import ( ) type blockEnc struct { - size int - literals []byte - sequences []seq - coders seqCoders - litEnc *huff0.Scratch - wr bitWriter + size int + literals []byte + sequences []seq + coders seqCoders + litEnc *huff0.Scratch + dictLitEnc *huff0.Scratch + wr bitWriter extraLits int last bool @@ -75,6 +76,7 @@ func (b *blockEnc) reset(prev *blockEnc) { if prev != nil { b.recentOffsets = prev.prevRecentOffsets } + b.dictLitEnc = nil } // reset will reset the block for a new encode, but in the same stream, @@ -295,7 +297,7 @@ func (b *blockEnc) encodeRaw(a []byte) { b.output = bh.appendTo(b.output[:0]) b.output = append(b.output, a...) if debug { - println("Adding RAW block, length", len(a)) + println("Adding RAW block, length", len(a), "last:", b.last) } } @@ -308,25 +310,25 @@ func (b *blockEnc) encodeRawTo(dst, src []byte) []byte { dst = bh.appendTo(dst) dst = append(dst, src...) if debug { - println("Adding RAW block, length", len(src)) + println("Adding RAW block, length", len(src), "last:", b.last) } return dst } // encodeLits can be used if the block is only litLen. -func (b *blockEnc) encodeLits(raw bool) error { +func (b *blockEnc) encodeLits(lits []byte, raw bool) error { var bh blockHeader bh.setLast(b.last) - bh.setSize(uint32(len(b.literals))) + bh.setSize(uint32(len(lits))) // Don't compress extremely small blocks - if len(b.literals) < 32 || raw { + if len(lits) < 8 || (len(lits) < 32 && b.dictLitEnc == nil) || raw { if debug { - println("Adding RAW block, length", len(b.literals)) + println("Adding RAW block, length", len(lits), "last:", b.last) } bh.setType(blockTypeRaw) b.output = bh.appendTo(b.output) - b.output = append(b.output, b.literals...) + b.output = append(b.output, lits...) return nil } @@ -335,13 +337,18 @@ func (b *blockEnc) encodeLits(raw bool) error { reUsed, single bool err error ) - if len(b.literals) >= 1024 { + if b.dictLitEnc != nil { + b.litEnc.TransferCTable(b.dictLitEnc) + b.litEnc.Reuse = huff0.ReusePolicyAllow + b.dictLitEnc = nil + } + if len(lits) >= 1024 { // Use 4 Streams. - out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) - } else if len(b.literals) > 32 { + out, reUsed, err = huff0.Compress4X(lits, b.litEnc) + } else if len(lits) > 32 { // Use 1 stream single = true - out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc) + out, reUsed, err = huff0.Compress1X(lits, b.litEnc) } else { err = huff0.ErrIncompressible } @@ -349,19 +356,19 @@ func (b *blockEnc) encodeLits(raw bool) error { switch err { case huff0.ErrIncompressible: if debug { - println("Adding RAW block, length", len(b.literals)) + println("Adding RAW block, length", len(lits), "last:", b.last) } bh.setType(blockTypeRaw) b.output = bh.appendTo(b.output) - b.output = append(b.output, b.literals...) + b.output = append(b.output, lits...) return nil case huff0.ErrUseRLE: if debug { - println("Adding RLE block, length", len(b.literals)) + println("Adding RLE block, length", len(lits)) } bh.setType(blockTypeRLE) b.output = bh.appendTo(b.output) - b.output = append(b.output, b.literals[0]) + b.output = append(b.output, lits[0]) return nil default: return err @@ -384,7 +391,7 @@ func (b *blockEnc) encodeLits(raw bool) error { lh.setType(literalsBlockCompressed) } // Set sizes - lh.setSizes(len(out), len(b.literals), single) + lh.setSizes(len(out), len(lits), single) bh.setSize(uint32(len(out) + lh.size() + 1)) // Write block headers. @@ -444,13 +451,19 @@ func fuzzFseEncoder(data []byte) int { } // encode will encode the block and append the output in b.output. -func (b *blockEnc) encode(raw, rawAllLits bool) error { +// Previous offset codes must be pushed if more blocks are expected. +func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if len(b.sequences) == 0 { - return b.encodeLits(rawAllLits) + return b.encodeLits(b.literals, rawAllLits) } - // We want some difference - if len(b.literals) > (b.size - (b.size >> 5)) { - return errIncompressible + // We want some difference to at least account for the headers. + saved := b.size - len(b.literals) - (b.size >> 5) + if saved < 16 { + if org == nil { + return errIncompressible + } + b.popOffsets() + return b.encodeLits(org, rawAllLits) } var bh blockHeader @@ -466,6 +479,11 @@ func (b *blockEnc) encode(raw, rawAllLits bool) error { reUsed, single bool err error ) + if b.dictLitEnc != nil { + b.litEnc.TransferCTable(b.dictLitEnc) + b.litEnc.Reuse = huff0.ReusePolicyAllow + b.dictLitEnc = nil + } if len(b.literals) >= 1024 && !raw { // Use 4 Streams. out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go new file mode 100644 index 0000000000000..87896c5eaa789 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/decodeheader.go @@ -0,0 +1,202 @@ +// Copyright 2020+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. + +package zstd + +import ( + "bytes" + "errors" + "io" +) + +// HeaderMaxSize is the maximum size of a Frame and Block Header. +// If less is sent to Header.Decode it *may* still contain enough information. +const HeaderMaxSize = 14 + 3 + +// Header contains information about the first frame and block within that. +type Header struct { + // Window Size the window of data to keep while decoding. + // Will only be set if HasFCS is false. + WindowSize uint64 + + // Frame content size. + // Expected size of the entire frame. + FrameContentSize uint64 + + // Dictionary ID. + // If 0, no dictionary. + DictionaryID uint32 + + // First block information. + FirstBlock struct { + // OK will be set if first block could be decoded. + OK bool + + // Is this the last block of a frame? + Last bool + + // Is the data compressed? + // If true CompressedSize will be populated. + // Unfortunately DecompressedSize cannot be determined + // without decoding the blocks. + Compressed bool + + // DecompressedSize is the expected decompressed size of the block. + // Will be 0 if it cannot be determined. + DecompressedSize int + + // CompressedSize of the data in the block. + // Does not include the block header. + // Will be equal to DecompressedSize if not Compressed. + CompressedSize int + } + + // Skippable will be true if the frame is meant to be skipped. + // No other information will be populated. + Skippable bool + + // If set there is a checksum present for the block content. + HasCheckSum bool + + // If this is true FrameContentSize will have a valid value + HasFCS bool + + SingleSegment bool +} + +// Decode the header from the beginning of the stream. +// This will decode the frame header and the first block header if enough bytes are provided. +// It is recommended to provide at least HeaderMaxSize bytes. +// If the frame header cannot be read an error will be returned. +// If there isn't enough input, io.ErrUnexpectedEOF is returned. +// The FirstBlock.OK will indicate if enough information was available to decode the first block header. +func (h *Header) Decode(in []byte) error { + if len(in) < 4 { + return io.ErrUnexpectedEOF + } + b, in := in[:4], in[4:] + if !bytes.Equal(b, frameMagic) { + if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 { + return ErrMagicMismatch + } + *h = Header{Skippable: true} + return nil + } + if len(in) < 1 { + return io.ErrUnexpectedEOF + } + + // Clear output + *h = Header{} + fhd, in := in[0], in[1:] + h.SingleSegment = fhd&(1<<5) != 0 + h.HasCheckSum = fhd&(1<<2) != 0 + + if fhd&(1<<3) != 0 { + return errors.New("Reserved bit set on frame header") + } + + // Read Window_Descriptor + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor + if !h.SingleSegment { + if len(in) < 1 { + return io.ErrUnexpectedEOF + } + var wd byte + wd, in = in[0], in[1:] + windowLog := 10 + (wd >> 3) + windowBase := uint64(1) << windowLog + windowAdd := (windowBase / 8) * uint64(wd&0x7) + h.WindowSize = windowBase + windowAdd + } + + // Read Dictionary_ID + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id + if size := fhd & 3; size != 0 { + if size == 3 { + size = 4 + } + if len(in) < int(size) { + return io.ErrUnexpectedEOF + } + b, in = in[:size], in[size:] + if b == nil { + return io.ErrUnexpectedEOF + } + switch size { + case 1: + h.DictionaryID = uint32(b[0]) + case 2: + h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) + case 4: + h.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + } + } + + // Read Frame_Content_Size + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size + var fcsSize int + v := fhd >> 6 + switch v { + case 0: + if h.SingleSegment { + fcsSize = 1 + } + default: + fcsSize = 1 << v + } + + if fcsSize > 0 { + h.HasFCS = true + if len(in) < fcsSize { + return io.ErrUnexpectedEOF + } + b, in = in[:fcsSize], in[fcsSize:] + if b == nil { + return io.ErrUnexpectedEOF + } + switch fcsSize { + case 1: + h.FrameContentSize = uint64(b[0]) + case 2: + // When FCS_Field_Size is 2, the offset of 256 is added. + h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) + 256 + case 4: + h.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) | (uint64(b[2]) << 16) | (uint64(b[3]) << 24) + case 8: + d1 := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24) + h.FrameContentSize = uint64(d1) | (uint64(d2) << 32) + } + } + + // Frame Header done, we will not fail from now on. + if len(in) < 3 { + return nil + } + tmp, in := in[:3], in[3:] + bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) + h.FirstBlock.Last = bh&1 != 0 + blockType := blockType((bh >> 1) & 3) + // find size. + cSize := int(bh >> 3) + switch blockType { + case blockTypeReserved: + return nil + case blockTypeRLE: + h.FirstBlock.Compressed = true + h.FirstBlock.DecompressedSize = cSize + h.FirstBlock.CompressedSize = 1 + case blockTypeCompressed: + h.FirstBlock.Compressed = true + h.FirstBlock.CompressedSize = cSize + case blockTypeRaw: + h.FirstBlock.DecompressedSize = cSize + h.FirstBlock.CompressedSize = cSize + default: + panic("Invalid block type") + } + + h.FirstBlock.OK = true + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 66b51bf2d3488..cdda0de58b447 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -187,7 +187,7 @@ func (d *Decoder) Reset(r io.Reader) error { d.current.err = err d.current.flushed = true if debug { - println("sync decode to ", len(dst), "bytes, err:", err) + println("sync decode to", len(dst), "bytes, err:", err) } return nil } @@ -303,6 +303,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.history.reset() err := frame.reset(&frame.bBuf) if err == io.EOF { + if debug { + println("frame reset return EOF") + } return dst, nil } if frame.DictionaryID != nil { @@ -320,19 +323,23 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } if frame.FrameContentSize > 0 && frame.FrameContentSize < 1<<30 { // Never preallocate moe than 1 GB up front. - if uint64(cap(dst)) < frame.FrameContentSize { + if cap(dst)-len(dst) < int(frame.FrameContentSize) { dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)) copy(dst2, dst) dst = dst2 } } if cap(dst) == 0 { - // Allocate window size * 2 by default if nothing is provided and we didn't get frame content size. - size := frame.WindowSize * 2 + // Allocate len(input) * 2 by default if nothing is provided + // and we didn't get frame content size. + size := len(input) * 2 // Cap to 1 MB. if size > 1<<20 { size = 1 << 20 } + if uint64(size) > d.o.maxDecodedSize { + size = int(d.o.maxDecodedSize) + } dst = make([]byte, 0, size) } @@ -341,6 +348,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { return dst, err } if len(frame.bBuf) == 0 { + if debug { + println("frame dbuf empty") + } break } } diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 8eb6f6ba330a5..fa25a18d8644e 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -13,14 +13,31 @@ import ( type dict struct { id uint32 - litDec *huff0.Scratch + litEnc *huff0.Scratch llDec, ofDec, mlDec sequenceDec - offsets [3]int - content []byte + //llEnc, ofEnc, mlEnc []*fseEncoder + offsets [3]int + content []byte } var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} +// ID returns the dictionary id or 0 if d is nil. +func (d *dict) ID() uint32 { + if d == nil { + return 0 + } + return d.id +} + +// DictContentSize returns the dictionary content size or 0 if d is nil. +func (d *dict) DictContentSize() int { + if d == nil { + return 0 + } + return len(d.content) +} + // Load a dictionary as described in // https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format func loadDict(b []byte) (*dict, error) { @@ -43,10 +60,11 @@ func loadDict(b []byte) (*dict, error) { // Read literal table var err error - d.litDec, b, err = huff0.ReadTable(b[8:], nil) + d.litEnc, b, err = huff0.ReadTable(b[8:], nil) if err != nil { return nil, err } + d.litEnc.Reuse = huff0.ReusePolicyMust br := byteReader{ b: b, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go new file mode 100644 index 0000000000000..b1b7c6e6a72d0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -0,0 +1,155 @@ +package zstd + +import ( + "fmt" + "math/bits" + + "github.com/klauspost/compress/zstd/internal/xxhash" +) + +type fastBase struct { + // cur is the offset at the start of hist + cur int32 + // maximum offset. Should be at least 2x block size. + maxMatchOff int32 + hist []byte + crc *xxhash.Digest + tmp [8]byte + blk *blockEnc + lastDictID uint32 +} + +// CRC returns the underlying CRC writer. +func (e *fastBase) CRC() *xxhash.Digest { + return e.crc +} + +// AppendCRC will append the CRC to the destination slice and return it. +func (e *fastBase) AppendCRC(dst []byte) []byte { + crc := e.crc.Sum(e.tmp[:0]) + dst = append(dst, crc[7], crc[6], crc[5], crc[4]) + return dst +} + +// WindowSize returns the window size of the encoder, +// or a window size small enough to contain the input size, if > 0. +func (e *fastBase) WindowSize(size int) int32 { + if size > 0 && size < int(e.maxMatchOff) { + b := int32(1) << uint(bits.Len(uint(size))) + // Keep minimum window. + if b < 1024 { + b = 1024 + } + return b + } + return e.maxMatchOff +} + +// Block returns the current block. +func (e *fastBase) Block() *blockEnc { + return e.blk +} + +func (e *fastBase) addBlock(src []byte) int32 { + if debugAsserts && e.cur > bufferReset { + panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset)) + } + // check if we have space already + if len(e.hist)+len(src) > cap(e.hist) { + if cap(e.hist) == 0 { + l := e.maxMatchOff * 2 + // Make it at least 1MB. + if l < 1<<20 { + l = 1 << 20 + } + e.hist = make([]byte, 0, l) + } else { + if cap(e.hist) < int(e.maxMatchOff*2) { + panic("unexpected buffer size") + } + // Move down + offset := int32(len(e.hist)) - e.maxMatchOff + copy(e.hist[0:e.maxMatchOff], e.hist[offset:]) + e.cur += offset + e.hist = e.hist[:e.maxMatchOff] + } + } + s := int32(len(e.hist)) + e.hist = append(e.hist, src...) + return s +} + +// useBlock will replace the block with the provided one, +// but transfer recent offsets from the previous. +func (e *fastBase) UseBlock(enc *blockEnc) { + enc.reset(e.blk) + e.blk = enc +} + +func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 { + // Extend the match to be as long as possible. + return int32(matchLen(src[s:], src[t:])) +} + +func (e *fastBase) matchlen(s, t int32, src []byte) int32 { + if debugAsserts { + if s < 0 { + err := fmt.Sprintf("s (%d) < 0", s) + panic(err) + } + if t < 0 { + err := fmt.Sprintf("s (%d) < 0", s) + panic(err) + } + if s-t > e.maxMatchOff { + err := fmt.Sprintf("s (%d) - t (%d) > maxMatchOff (%d)", s, t, e.maxMatchOff) + panic(err) + } + if len(src)-int(s) > maxCompressedBlockSize { + panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) + } + } + + // Extend the match to be as long as possible. + return int32(matchLen(src[s:], src[t:])) +} + +// Reset the encoding table. +func (e *fastBase) resetBase(d *dict, singleBlock bool) { + if e.blk == nil { + e.blk = &blockEnc{} + e.blk.init() + } else { + e.blk.reset(nil) + } + e.blk.initNewEncode() + if e.crc == nil { + e.crc = xxhash.New() + } else { + e.crc.Reset() + } + if (!singleBlock || d.DictContentSize() > 0) && cap(e.hist) < int(e.maxMatchOff*2)+d.DictContentSize() { + l := e.maxMatchOff*2 + int32(d.DictContentSize()) + // Make it at least 1MB. + if l < 1<<20 { + l = 1 << 20 + } + e.hist = make([]byte, 0, l) + } + // We offset current position so everything will be out of reach. + // If above reset line, history will be purged. + if e.cur < bufferReset { + e.cur += e.maxMatchOff + int32(len(e.hist)) + } + e.hist = e.hist[:0] + if d != nil { + // Set offsets (currently not used) + for i, off := range d.offsets { + e.blk.recentOffsets[i] = uint32(off) + e.blk.prevRecentOffsets[i] = e.blk.recentOffsets[i] + } + // Transfer litenc. + e.blk.dictLitEnc = d.litEnc + e.hist = append(e.hist, d.content...) + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go new file mode 100644 index 0000000000000..c4baa42c6484b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -0,0 +1,484 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "fmt" + "math/bits" +) + +const ( + bestLongTableBits = 20 // Bits used in the long match table + bestLongTableSize = 1 << bestLongTableBits // Size of the table + + // Note: Increasing the short table bits or making the hash shorter + // can actually lead to compression degradation since it will 'steal' more from the + // long match table and match offsets are quite big. + // This greatly depends on the type of input. + bestShortTableBits = 16 // Bits used in the short match table + bestShortTableSize = 1 << bestShortTableBits // Size of the table +) + +// bestFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. +// The long match table contains the previous entry with the same hash, +// effectively making it a "chain" of length 2. +// When we find a long match we choose between the two values and select the longest. +// When we find a short match, after checking the long, we check if we can find a long at n+1 +// and that it is longer (lazy matching). +type bestFastEncoder struct { + fastBase + table [bestShortTableSize]prevEntry + longTable [bestLongTableSize]prevEntry + dictTable []prevEntry + dictLongTable []prevEntry +} + +// Encode improves compression... +func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 4 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = prevEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = prevEntry{} + } + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + v2 := e.table[i].prev + if v < minOff { + v = 0 + v2 = 0 + } else { + v = v - e.cur + e.maxMatchOff + if v2 < minOff { + v2 = 0 + } else { + v2 = v2 - e.cur + e.maxMatchOff + } + } + e.table[i] = prevEntry{ + offset: v, + prev: v2, + } + } + for i := range e.longTable[:] { + v := e.longTable[i].offset + v2 := e.longTable[i].prev + if v < minOff { + v = 0 + v2 = 0 + } else { + v = v - e.cur + e.maxMatchOff + if v2 < minOff { + v2 = 0 + } else { + v2 = v2 - e.cur + e.maxMatchOff + } + } + e.longTable[i] = prevEntry{ + offset: v, + prev: v2, + } + } + e.cur = e.maxMatchOff + break + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + const kSearchStrength = 12 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + offset3 := int32(blk.recentOffsets[2]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + _ = addLiterals + + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + // We allow the encoder to optionally turn off repeat offsets across blocks + canRepeat := len(blk.sequences) > 2 + + if debugAsserts && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + type match struct { + offset int32 + s int32 + length int32 + rep int32 + } + matchAt := func(offset int32, s int32, first uint32, rep int32) match { + if s-offset >= e.maxMatchOff || load3232(src, offset) != first { + return match{offset: offset, s: s} + } + return match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} + } + + bestOf := func(a, b match) match { + aScore := b.s - a.s + a.length + bScore := a.s - b.s + b.length + if a.rep < 0 { + aScore = aScore - int32(bits.Len32(uint32(a.offset)))/8 + } + if b.rep < 0 { + bScore = bScore - int32(bits.Len32(uint32(b.offset)))/8 + } + if aScore >= bScore { + return a + } + return b + } + const goodEnough = 100 + + nextHashL := hash8(cv, bestLongTableBits) + nextHashS := hash4x64(cv, bestShortTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) + best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)) + if canRepeat && best.length < goodEnough { + best = bestOf(best, matchAt(s-offset1+1, s+1, uint32(cv>>8), 1)) + best = bestOf(best, matchAt(s-offset2+1, s+1, uint32(cv>>8), 2)) + best = bestOf(best, matchAt(s-offset3+1, s+1, uint32(cv>>8), 3)) + best = bestOf(best, matchAt(s-offset1+3, s+3, uint32(cv>>24), 1)) + best = bestOf(best, matchAt(s-offset2+3, s+3, uint32(cv>>24), 2)) + best = bestOf(best, matchAt(s-offset3+3, s+3, uint32(cv>>24), 3)) + } + // Load next and check... + e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: candidateL.offset} + e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: candidateS.offset} + + // Look far ahead, unless we have a really long match already... + if best.length < goodEnough { + // No match found, move forward on input, no need to check forward... + if best.length < 4 { + s += 1 + (s-nextEmit)>>(kSearchStrength-1) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + continue + } + + s++ + candidateS = e.table[hash4x64(cv>>8, bestShortTableBits)] + cv = load6432(src, s) + cv2 := load6432(src, s+1) + candidateL = e.longTable[hash8(cv, bestLongTableBits)] + candidateL2 := e.longTable[hash8(cv2, bestLongTableBits)] + + best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)) + best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) + best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) + best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) + } + + // We have a match, we can store the forward value + if best.rep > 0 { + s = best.s + var seq seq + seq.matchLen = uint32(best.length - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := best.s + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + repIndex := best.offset + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = uint32(best.rep) + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Index match start+1 (long) -> s - 1 + index0 := s + s = best.s + best.length + + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, best.length) + + } + break encodeLoop + } + // Index skipped... + off := index0 + e.cur + for index0 < s-1 { + cv0 := load6432(src, index0) + h0 := hash8(cv0, bestLongTableBits) + h1 := hash4x64(cv0, bestShortTableBits) + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} + off++ + index0++ + } + switch best.rep { + case 2: + offset1, offset2 = offset2, offset1 + case 3: + offset1, offset2, offset3 = offset3, offset1, offset2 + } + cv = load6432(src, s) + continue + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + s = best.s + t := best.offset + offset1, offset2, offset3 = s-t, offset1, offset2 + + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + + if debugAsserts && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the n-byte match as long as possible. + l := best.length + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) -> s - 1 + index0 := s - l + 1 + // every entry + for index0 < s-1 { + cv0 := load6432(src, index0) + h0 := hash8(cv0, bestLongTableBits) + h1 := hash4x64(cv0, bestShortTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} + index0++ + } + + cv = load6432(src, s) + if !canRepeat { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash4x64(cv, bestShortTableBits) + nextHashL := hash8(cv, bestLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} + e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: e.table[nextHashS].offset} + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + blk.recentOffsets[2] = uint32(offset3) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + +// EncodeNoHist will encode a block with no history and no following blocks. +// Most notable difference is that src will not be copied for history and +// we do not need to check for max match length. +func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + e.Encode(blk, src) +} + +// ResetDict will reset and set a dictionary if not nil +func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { + e.resetBase(d, singleBlock) + if d == nil { + return + } + // Init or copy dict table + if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) { + e.dictTable = make([]prevEntry, len(e.table)) + } + end := int32(len(d.content)) - 8 + e.maxMatchOff + for i := e.maxMatchOff; i < end; i += 4 { + const hashLog = bestShortTableBits + + cv := load6432(d.content, i-e.maxMatchOff) + nextHash := hash4x64(cv, hashLog) // 0 -> 4 + nextHash1 := hash4x64(cv>>8, hashLog) // 1 -> 5 + nextHash2 := hash4x64(cv>>16, hashLog) // 2 -> 6 + nextHash3 := hash4x64(cv>>24, hashLog) // 3 -> 7 + e.dictTable[nextHash] = prevEntry{ + prev: e.dictTable[nextHash].offset, + offset: i, + } + e.dictTable[nextHash1] = prevEntry{ + prev: e.dictTable[nextHash1].offset, + offset: i + 1, + } + e.dictTable[nextHash2] = prevEntry{ + prev: e.dictTable[nextHash2].offset, + offset: i + 2, + } + e.dictTable[nextHash3] = prevEntry{ + prev: e.dictTable[nextHash3].offset, + offset: i + 3, + } + } + e.lastDictID = d.id + } + + // Init or copy dict table + if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) { + e.dictLongTable = make([]prevEntry, len(e.longTable)) + } + if len(d.content) >= 8 { + cv := load6432(d.content, 0) + h := hash8(cv, bestLongTableBits) + e.dictLongTable[h] = prevEntry{ + offset: e.maxMatchOff, + prev: e.dictLongTable[h].offset, + } + + end := int32(len(d.content)) - 8 + e.maxMatchOff + off := 8 // First to read + for i := e.maxMatchOff + 1; i < end; i++ { + cv = cv>>8 | (uint64(d.content[off]) << 56) + h := hash8(cv, bestLongTableBits) + e.dictLongTable[h] = prevEntry{ + offset: i, + prev: e.dictLongTable[h].offset, + } + off++ + } + } + e.lastDictID = d.id + } + // Reset table to initial state + copy(e.longTable[:], e.dictLongTable) + + e.cur = e.maxMatchOff + // Reset table to initial state + copy(e.table[:], e.dictTable) +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index c120d905481ac..94a5343d00ecb 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -31,8 +31,10 @@ type prevEntry struct { // and that it is longer (lazy matching). type betterFastEncoder struct { fastBase - table [betterShortTableSize]tableEntry - longTable [betterLongTableSize]prevEntry + table [betterShortTableSize]tableEntry + longTable [betterLongTableSize]prevEntry + dictTable []tableEntry + dictLongTable []prevEntry } // Encode improves compression... @@ -516,3 +518,78 @@ encodeLoop: func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { e.Encode(blk, src) } + +// ResetDict will reset and set a dictionary if not nil +func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { + e.resetBase(d, singleBlock) + if d == nil { + return + } + // Init or copy dict table + if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) { + e.dictTable = make([]tableEntry, len(e.table)) + } + end := int32(len(d.content)) - 8 + e.maxMatchOff + for i := e.maxMatchOff; i < end; i += 4 { + const hashLog = betterShortTableBits + + cv := load6432(d.content, i-e.maxMatchOff) + nextHash := hash5(cv, hashLog) // 0 -> 4 + nextHash1 := hash5(cv>>8, hashLog) // 1 -> 5 + nextHash2 := hash5(cv>>16, hashLog) // 2 -> 6 + nextHash3 := hash5(cv>>24, hashLog) // 3 -> 7 + e.dictTable[nextHash] = tableEntry{ + val: uint32(cv), + offset: i, + } + e.dictTable[nextHash1] = tableEntry{ + val: uint32(cv >> 8), + offset: i + 1, + } + e.dictTable[nextHash2] = tableEntry{ + val: uint32(cv >> 16), + offset: i + 2, + } + e.dictTable[nextHash3] = tableEntry{ + val: uint32(cv >> 24), + offset: i + 3, + } + } + e.lastDictID = d.id + } + + // Init or copy dict table + if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) { + e.dictLongTable = make([]prevEntry, len(e.longTable)) + } + if len(d.content) >= 8 { + cv := load6432(d.content, 0) + h := hash8(cv, betterLongTableBits) + e.dictLongTable[h] = prevEntry{ + offset: e.maxMatchOff, + prev: e.dictLongTable[h].offset, + } + + end := int32(len(d.content)) - 8 + e.maxMatchOff + off := 8 // First to read + for i := e.maxMatchOff + 1; i < end; i++ { + cv = cv>>8 | (uint64(d.content[off]) << 56) + h := hash8(cv, betterLongTableBits) + e.dictLongTable[h] = prevEntry{ + offset: i, + prev: e.dictLongTable[h].offset, + } + off++ + } + } + e.lastDictID = d.id + } + // Reset table to initial state + copy(e.longTable[:], e.dictLongTable) + + e.cur = e.maxMatchOff + // Reset table to initial state + copy(e.table[:], e.dictTable) +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 50276bcde7921..19eebf66e5024 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -18,7 +18,8 @@ const ( type doubleFastEncoder struct { fastEncoder - longTable [dFastLongTableSize]tableEntry + longTable [dFastLongTableSize]tableEntry + dictLongTable []tableEntry } // Encode mimmics functionality in zstd_dfast.c @@ -494,7 +495,7 @@ encodeLoop: // but the likelihood of both the first 4 bytes and the hash matching should be enough. t = candidateL.offset - e.cur if debugAsserts && s <= t { - panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + panic(fmt.Sprintf("s (%d) <= t (%d). cur: %d", s, t, e.cur)) } if debugAsserts && s-t > e.maxMatchOff { panic("s - t >e.maxMatchOff") @@ -676,3 +677,37 @@ encodeLoop: e.cur += int32(len(src)) } } + +// ResetDict will reset and set a dictionary if not nil +func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { + e.fastEncoder.Reset(d, singleBlock) + if d == nil { + return + } + + // Init or copy dict table + if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) { + e.dictLongTable = make([]tableEntry, len(e.longTable)) + } + if len(d.content) >= 8 { + cv := load6432(d.content, 0) + e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + val: uint32(cv), + offset: e.maxMatchOff, + } + end := int32(len(d.content)) - 8 + e.maxMatchOff + for i := e.maxMatchOff + 1; i < end; i++ { + cv = cv>>8 | (uint64(d.content[i-e.maxMatchOff+7]) << 56) + e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + val: uint32(cv), + offset: i, + } + } + } + e.lastDictID = d.id + } + // Reset table to initial state + e.cur = e.maxMatchOff + copy(e.longTable[:], e.dictLongTable) +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 4104b456ce2e7..0b301df439088 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -8,8 +8,6 @@ import ( "fmt" "math" "math/bits" - - "github.com/klauspost/compress/zstd/internal/xxhash" ) const ( @@ -24,51 +22,10 @@ type tableEntry struct { offset int32 } -type fastBase struct { - // cur is the offset at the start of hist - cur int32 - // maximum offset. Should be at least 2x block size. - maxMatchOff int32 - hist []byte - crc *xxhash.Digest - tmp [8]byte - blk *blockEnc -} - type fastEncoder struct { fastBase - table [tableSize]tableEntry -} - -// CRC returns the underlying CRC writer. -func (e *fastBase) CRC() *xxhash.Digest { - return e.crc -} - -// AppendCRC will append the CRC to the destination slice and return it. -func (e *fastBase) AppendCRC(dst []byte) []byte { - crc := e.crc.Sum(e.tmp[:0]) - dst = append(dst, crc[7], crc[6], crc[5], crc[4]) - return dst -} - -// WindowSize returns the window size of the encoder, -// or a window size small enough to contain the input size, if > 0. -func (e *fastBase) WindowSize(size int) int32 { - if size > 0 && size < int(e.maxMatchOff) { - b := int32(1) << uint(bits.Len(uint(size))) - // Keep minimum window. - if b < 1024 { - b = 1024 - } - return b - } - return e.maxMatchOff -} - -// Block returns the current block. -func (e *fastBase) Block() *blockEnc { - return e.blk + table [tableSize]tableEntry + dictTable []tableEntry } // Encode mimmics functionality in zstd_fast.c @@ -660,96 +617,45 @@ encodeLoop: } } -func (e *fastBase) addBlock(src []byte) int32 { - if debugAsserts && e.cur > bufferReset { - panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset)) - } - // check if we have space already - if len(e.hist)+len(src) > cap(e.hist) { - if cap(e.hist) == 0 { - l := e.maxMatchOff * 2 - // Make it at least 1MB. - if l < 1<<20 { - l = 1 << 20 - } - e.hist = make([]byte, 0, l) - } else { - if cap(e.hist) < int(e.maxMatchOff*2) { - panic("unexpected buffer size") - } - // Move down - offset := int32(len(e.hist)) - e.maxMatchOff - copy(e.hist[0:e.maxMatchOff], e.hist[offset:]) - e.cur += offset - e.hist = e.hist[:e.maxMatchOff] - } +// ResetDict will reset and set a dictionary if not nil +func (e *fastEncoder) Reset(d *dict, singleBlock bool) { + e.resetBase(d, singleBlock) + if d == nil { + return } - s := int32(len(e.hist)) - e.hist = append(e.hist, src...) - return s -} -// useBlock will replace the block with the provided one, -// but transfer recent offsets from the previous. -func (e *fastBase) UseBlock(enc *blockEnc) { - enc.reset(e.blk) - e.blk = enc -} - -func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 { - // Extend the match to be as long as possible. - return int32(matchLen(src[s:], src[t:])) -} - -func (e *fastBase) matchlen(s, t int32, src []byte) int32 { - if debugAsserts { - if s < 0 { - err := fmt.Sprintf("s (%d) < 0", s) - panic(err) - } - if t < 0 { - err := fmt.Sprintf("s (%d) < 0", s) - panic(err) - } - if s-t > e.maxMatchOff { - err := fmt.Sprintf("s (%d) - t (%d) > maxMatchOff (%d)", s, t, e.maxMatchOff) - panic(err) - } - if len(src)-int(s) > maxCompressedBlockSize { - panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) + // Init or copy dict table + if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) { + e.dictTable = make([]tableEntry, len(e.table)) + } + if true { + end := e.maxMatchOff + int32(len(d.content)) - 8 + for i := e.maxMatchOff; i < end; i += 3 { + const hashLog = tableBits + + cv := load6432(d.content, i-e.maxMatchOff) + nextHash := hash6(cv, hashLog) // 0 -> 5 + nextHash1 := hash6(cv>>8, hashLog) // 1 -> 6 + nextHash2 := hash6(cv>>16, hashLog) // 2 -> 7 + e.dictTable[nextHash] = tableEntry{ + val: uint32(cv), + offset: i, + } + e.dictTable[nextHash1] = tableEntry{ + val: uint32(cv >> 8), + offset: i + 1, + } + e.dictTable[nextHash2] = tableEntry{ + val: uint32(cv >> 16), + offset: i + 2, + } + } } + e.lastDictID = d.id } - // Extend the match to be as long as possible. - return int32(matchLen(src[s:], src[t:])) -} - -// Reset the encoding table. -func (e *fastBase) Reset(singleBlock bool) { - if e.blk == nil { - e.blk = &blockEnc{} - e.blk.init() - } else { - e.blk.reset(nil) - } - e.blk.initNewEncode() - if e.crc == nil { - e.crc = xxhash.New() - } else { - e.crc.Reset() - } - if !singleBlock && cap(e.hist) < int(e.maxMatchOff*2) { - l := e.maxMatchOff * 2 - // Make it at least 1MB. - if l < 1<<20 { - l = 1 << 20 - } - e.hist = make([]byte, 0, l) - } - // We offset current position so everything will be out of reach. - // If above reset line, history will be purged. - if e.cur < bufferReset { - e.cur += e.maxMatchOff + int32(len(e.hist)) - } - e.hist = e.hist[:0] + e.cur = e.maxMatchOff + // Reset table to initial state + copy(e.table[:], e.dictTable) } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_params.go b/vendor/github.com/klauspost/compress/zstd/enc_params.go deleted file mode 100644 index d874116f715a5..0000000000000 --- a/vendor/github.com/klauspost/compress/zstd/enc_params.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -/* -// encParams are not really used, just here for reference. -type encParams struct { - // largest match distance : larger == more compression, more memory needed during decompression - windowLog uint8 - - // fully searched segment : larger == more compression, slower, more memory (useless for fast) - chainLog uint8 - - // dispatch table : larger == faster, more memory - hashLog uint8 - - // < nb of searches : larger == more compression, slower - searchLog uint8 - - // < match length searched : larger == faster decompression, sometimes less compression - minMatch uint8 - - // acceptable match size for optimal parser (only) : larger == more compression, slower - targetLength uint32 - - // see ZSTD_strategy definition above - strategy strategy -} - -// strategy defines the algorithm to use when generating sequences. -type strategy uint8 - -const ( - // Compression strategies, listed from fastest to strongest - strategyFast strategy = iota + 1 - strategyDfast - strategyGreedy - strategyLazy - strategyLazy2 - strategyBtlazy2 - strategyBtopt - strategyBtultra - strategyBtultra2 - // note : new strategies _might_ be added in the future. - // Only the order (from fast to strong) is guaranteed - -) - -var defEncParams = [4][]encParams{ - { // "default" - for any srcSize > 256 KB - // W, C, H, S, L, TL, strat - {19, 12, 13, 1, 6, 1, strategyFast}, // base for negative levels - {19, 13, 14, 1, 7, 0, strategyFast}, // level 1 - {20, 15, 16, 1, 6, 0, strategyFast}, // level 2 - {21, 16, 17, 1, 5, 1, strategyDfast}, // level 3 - {21, 18, 18, 1, 5, 1, strategyDfast}, // level 4 - {21, 18, 19, 2, 5, 2, strategyGreedy}, // level 5 - {21, 19, 19, 3, 5, 4, strategyGreedy}, // level 6 - {21, 19, 19, 3, 5, 8, strategyLazy}, // level 7 - {21, 19, 19, 3, 5, 16, strategyLazy2}, // level 8 - {21, 19, 20, 4, 5, 16, strategyLazy2}, // level 9 - {22, 20, 21, 4, 5, 16, strategyLazy2}, // level 10 - {22, 21, 22, 4, 5, 16, strategyLazy2}, // level 11 - {22, 21, 22, 5, 5, 16, strategyLazy2}, // level 12 - {22, 21, 22, 5, 5, 32, strategyBtlazy2}, // level 13 - {22, 22, 23, 5, 5, 32, strategyBtlazy2}, // level 14 - {22, 23, 23, 6, 5, 32, strategyBtlazy2}, // level 15 - {22, 22, 22, 5, 5, 48, strategyBtopt}, // level 16 - {23, 23, 22, 5, 4, 64, strategyBtopt}, // level 17 - {23, 23, 22, 6, 3, 64, strategyBtultra}, // level 18 - {23, 24, 22, 7, 3, 256, strategyBtultra2}, // level 19 - {25, 25, 23, 7, 3, 256, strategyBtultra2}, // level 20 - {26, 26, 24, 7, 3, 512, strategyBtultra2}, // level 21 - {27, 27, 25, 9, 3, 999, strategyBtultra2}, // level 22 - }, - { // for srcSize <= 256 KB - // W, C, H, S, L, T, strat - {18, 12, 13, 1, 5, 1, strategyFast}, // base for negative levels - {18, 13, 14, 1, 6, 0, strategyFast}, // level 1 - {18, 14, 14, 1, 5, 1, strategyDfast}, // level 2 - {18, 16, 16, 1, 4, 1, strategyDfast}, // level 3 - {18, 16, 17, 2, 5, 2, strategyGreedy}, // level 4. - {18, 18, 18, 3, 5, 2, strategyGreedy}, // level 5. - {18, 18, 19, 3, 5, 4, strategyLazy}, // level 6. - {18, 18, 19, 4, 4, 4, strategyLazy}, // level 7 - {18, 18, 19, 4, 4, 8, strategyLazy2}, // level 8 - {18, 18, 19, 5, 4, 8, strategyLazy2}, // level 9 - {18, 18, 19, 6, 4, 8, strategyLazy2}, // level 10 - {18, 18, 19, 5, 4, 12, strategyBtlazy2}, // level 11. - {18, 19, 19, 7, 4, 12, strategyBtlazy2}, // level 12. - {18, 18, 19, 4, 4, 16, strategyBtopt}, // level 13 - {18, 18, 19, 4, 3, 32, strategyBtopt}, // level 14. - {18, 18, 19, 6, 3, 128, strategyBtopt}, // level 15. - {18, 19, 19, 6, 3, 128, strategyBtultra}, // level 16. - {18, 19, 19, 8, 3, 256, strategyBtultra}, // level 17. - {18, 19, 19, 6, 3, 128, strategyBtultra2}, // level 18. - {18, 19, 19, 8, 3, 256, strategyBtultra2}, // level 19. - {18, 19, 19, 10, 3, 512, strategyBtultra2}, // level 20. - {18, 19, 19, 12, 3, 512, strategyBtultra2}, // level 21. - {18, 19, 19, 13, 3, 999, strategyBtultra2}, // level 22. - }, - { // for srcSize <= 128 KB - // W, C, H, S, L, T, strat - {17, 12, 12, 1, 5, 1, strategyFast}, // base for negative levels - {17, 12, 13, 1, 6, 0, strategyFast}, // level 1 - {17, 13, 15, 1, 5, 0, strategyFast}, // level 2 - {17, 15, 16, 2, 5, 1, strategyDfast}, // level 3 - {17, 17, 17, 2, 4, 1, strategyDfast}, // level 4 - {17, 16, 17, 3, 4, 2, strategyGreedy}, // level 5 - {17, 17, 17, 3, 4, 4, strategyLazy}, // level 6 - {17, 17, 17, 3, 4, 8, strategyLazy2}, // level 7 - {17, 17, 17, 4, 4, 8, strategyLazy2}, // level 8 - {17, 17, 17, 5, 4, 8, strategyLazy2}, // level 9 - {17, 17, 17, 6, 4, 8, strategyLazy2}, // level 10 - {17, 17, 17, 5, 4, 8, strategyBtlazy2}, // level 11 - {17, 18, 17, 7, 4, 12, strategyBtlazy2}, // level 12 - {17, 18, 17, 3, 4, 12, strategyBtopt}, // level 13. - {17, 18, 17, 4, 3, 32, strategyBtopt}, // level 14. - {17, 18, 17, 6, 3, 256, strategyBtopt}, // level 15. - {17, 18, 17, 6, 3, 128, strategyBtultra}, // level 16. - {17, 18, 17, 8, 3, 256, strategyBtultra}, // level 17. - {17, 18, 17, 10, 3, 512, strategyBtultra}, // level 18. - {17, 18, 17, 5, 3, 256, strategyBtultra2}, // level 19. - {17, 18, 17, 7, 3, 512, strategyBtultra2}, // level 20. - {17, 18, 17, 9, 3, 512, strategyBtultra2}, // level 21. - {17, 18, 17, 11, 3, 999, strategyBtultra2}, // level 22. - }, - { // for srcSize <= 16 KB - // W, C, H, S, L, T, strat - {14, 12, 13, 1, 5, 1, strategyFast}, // base for negative levels - {14, 14, 15, 1, 5, 0, strategyFast}, // level 1 - {14, 14, 15, 1, 4, 0, strategyFast}, // level 2 - {14, 14, 15, 2, 4, 1, strategyDfast}, // level 3 - {14, 14, 14, 4, 4, 2, strategyGreedy}, // level 4 - {14, 14, 14, 3, 4, 4, strategyLazy}, // level 5. - {14, 14, 14, 4, 4, 8, strategyLazy2}, // level 6 - {14, 14, 14, 6, 4, 8, strategyLazy2}, // level 7 - {14, 14, 14, 8, 4, 8, strategyLazy2}, // level 8. - {14, 15, 14, 5, 4, 8, strategyBtlazy2}, // level 9. - {14, 15, 14, 9, 4, 8, strategyBtlazy2}, // level 10. - {14, 15, 14, 3, 4, 12, strategyBtopt}, // level 11. - {14, 15, 14, 4, 3, 24, strategyBtopt}, // level 12. - {14, 15, 14, 5, 3, 32, strategyBtultra}, // level 13. - {14, 15, 15, 6, 3, 64, strategyBtultra}, // level 14. - {14, 15, 15, 7, 3, 256, strategyBtultra}, // level 15. - {14, 15, 15, 5, 3, 48, strategyBtultra2}, // level 16. - {14, 15, 15, 6, 3, 128, strategyBtultra2}, // level 17. - {14, 15, 15, 7, 3, 256, strategyBtultra2}, // level 18. - {14, 15, 15, 8, 3, 256, strategyBtultra2}, // level 19. - {14, 15, 15, 8, 3, 512, strategyBtultra2}, // level 20. - {14, 15, 15, 9, 3, 512, strategyBtultra2}, // level 21. - {14, 15, 15, 10, 3, 999, strategyBtultra2}, // level 22. - }, -} -*/ diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index c56d2241f7e11..f5759211dac58 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -35,7 +35,7 @@ type encoder interface { AppendCRC([]byte) []byte WindowSize(size int) int32 UseBlock(*blockEnc) - Reset(singleBlock bool) + Reset(d *dict, singleBlock bool) } type encoderState struct { @@ -83,8 +83,6 @@ func (e *Encoder) initialize() { e.encoders = make(chan encoder, e.o.concurrent) for i := 0; i < e.o.concurrent; i++ { enc := e.o.encoder() - // If not single block, history will be allocated on first use. - enc.Reset(true) e.encoders <- enc } } @@ -115,7 +113,7 @@ func (e *Encoder) Reset(w io.Writer) { s.filling = s.filling[:0] s.current = s.current[:0] s.previous = s.previous[:0] - s.encoder.Reset(false) + s.encoder.Reset(e.o.dict, false) s.headerWritten = false s.eofWritten = false s.fullFrameWritten = false @@ -190,6 +188,7 @@ func (e *Encoder) nextBlock(final bool) error { s.filling = s.filling[:0] s.headerWritten = true s.fullFrameWritten = true + s.eofWritten = true return nil } @@ -199,8 +198,9 @@ func (e *Encoder) nextBlock(final bool) error { WindowSize: uint32(s.encoder.WindowSize(0)), SingleSegment: false, Checksum: e.o.crc, - DictID: 0, + DictID: e.o.dict.ID(), } + dst, err := fh.appendTo(tmp[:0]) if err != nil { return err @@ -280,7 +280,7 @@ func (e *Encoder) nextBlock(final bool) error { // If we got the exact same number of literals as input, // assume the literals cannot be compressed. if len(src) != len(blk.literals) || len(src) != e.o.blockSize { - err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy) + err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) } switch err { case errIncompressible: @@ -310,7 +310,13 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { if debug { println("Using ReadFrom") } - // Maybe handle stuff queued? + + // Flush any current writes. + if len(e.state.filling) > 0 { + if err := e.nextBlock(false); err != nil { + return 0, err + } + } e.state.filling = e.state.filling[:e.o.blockSize] src := e.state.filling for { @@ -327,7 +333,7 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { if debug { println("ReadFrom: got EOF final block:", len(e.state.filling)) } - return n, e.nextBlock(true) + return n, nil default: if debug { println("ReadFrom: got error:", err) @@ -449,7 +455,6 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { defer func() { // Release encoder reference to last block. // If a non-single block is needed the encoder will reset again. - enc.Reset(true) e.encoders <- enc }() // Use single segments when above minimum window and below 1MB. @@ -462,7 +467,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { WindowSize: uint32(enc.WindowSize(len(src))), SingleSegment: single, Checksum: e.o.crc, - DictID: 0, + DictID: e.o.dict.ID(), } // If less than 1MB, allocate a buffer up front. @@ -476,13 +481,18 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { // If we can do everything in one block, prefer that. if len(src) <= maxCompressedBlockSize { + enc.Reset(e.o.dict, true) // Slightly faster with no history and everything in one block. if e.o.crc { _, _ = enc.CRC().Write(src) } blk := enc.Block() blk.last = true - enc.EncodeNoHist(blk, src) + if e.o.dict == nil { + enc.EncodeNoHist(blk, src) + } else { + enc.Encode(blk, src) + } // If we got the exact same number of literals as input, // assume the literals cannot be compressed. @@ -491,7 +501,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { if len(blk.literals) != len(src) || len(src) != e.o.blockSize { // Output directly to dst blk.output = dst - err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy) + err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) } switch err { @@ -507,7 +517,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } blk.output = oldout } else { - enc.Reset(false) + enc.Reset(e.o.dict, false) blk := enc.Block() for len(src) > 0 { todo := src @@ -518,7 +528,6 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { if e.o.crc { _, _ = enc.CRC().Write(todo) } - blk.reset(nil) blk.pushOffsets() enc.Encode(blk, todo) if len(src) == 0 { @@ -528,7 +537,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { // If we got the exact same number of literals as input, // assume the literals cannot be compressed. if len(blk.literals) != len(todo) || len(todo) != e.o.blockSize { - err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy) + err = blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy) } switch err { @@ -543,6 +552,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { default: panic(err) } + blk.reset(nil) } } if e.o.crc { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index dfac14ddde9f8..a7312f42af7fe 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -24,17 +24,19 @@ type encoderOptions struct { allLitEntropy bool customWindow bool customALEntropy bool + dict *dict } func (o *encoderOptions) setDefault() { *o = encoderOptions{ // use less ram: true for now, but may change. - concurrent: runtime.GOMAXPROCS(0), - crc: true, - single: nil, - blockSize: 1 << 16, - windowSize: 8 << 20, - level: SpeedDefault, + concurrent: runtime.GOMAXPROCS(0), + crc: true, + single: nil, + blockSize: 1 << 16, + windowSize: 8 << 20, + level: SpeedDefault, + allLitEntropy: true, } } @@ -45,6 +47,8 @@ func (o encoderOptions) encoder() encoder { return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}} case SpeedBetterCompression: return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} + case SpeedBestCompression: + return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} case SpeedFastest: return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} } @@ -141,20 +145,20 @@ const ( // By using this, notice that CPU usage may go up in the future. SpeedBetterCompression + // SpeedBestCompression will choose the best available compression option. + // This will offer the best compression no matter the CPU cost. + SpeedBestCompression + // speedLast should be kept as the last actual compression option. // The is not for external usage, but is used to keep track of the valid options. speedLast - - // SpeedBestCompression will choose the best available compression option. - // For now this is not implemented. - SpeedBestCompression = SpeedBetterCompression ) // EncoderLevelFromString will convert a string representation of an encoding level back // to a compression level. The compare is not case sensitive. // If the string wasn't recognized, (false, SpeedDefault) will be returned. func EncoderLevelFromString(s string) (bool, EncoderLevel) { - for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ { + for l := speedNotSet + 1; l < speedLast; l++ { if strings.EqualFold(s, l.String()) { return true, l } @@ -171,7 +175,9 @@ func EncoderLevelFromZstd(level int) EncoderLevel { return SpeedFastest case level >= 3 && level < 6: return SpeedDefault - case level > 5: + case level >= 6 && level < 10: + return SpeedBetterCompression + case level >= 10: return SpeedBetterCompression } return SpeedDefault @@ -186,6 +192,8 @@ func (e EncoderLevel) String() string { return "default" case SpeedBetterCompression: return "better" + case SpeedBestCompression: + return "best" default: return "invalid" } @@ -207,6 +215,8 @@ func WithEncoderLevel(l EncoderLevel) EOption { o.windowSize = 8 << 20 case SpeedBetterCompression: o.windowSize = 16 << 20 + case SpeedBestCompression: + o.windowSize = 32 << 20 } } if !o.customALEntropy { @@ -265,3 +275,16 @@ func WithSingleSegment(b bool) EOption { return nil } } + +// WithEncoderDict allows to register a dictionary that will be used for the encode. +// The encoder *may* choose to use no dictionary instead for certain payloads. +func WithEncoderDict(dict []byte) EOption { + return func(o *encoderOptions) error { + d, err := loadDict(dict) + if err != nil { + return err + } + o.dict = d + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/frameenc.go b/vendor/github.com/klauspost/compress/zstd/frameenc.go index 4479cfe18b255..4ef7f5a3e3d53 100644 --- a/vendor/github.com/klauspost/compress/zstd/frameenc.go +++ b/vendor/github.com/klauspost/compress/zstd/frameenc.go @@ -5,6 +5,7 @@ package zstd import ( + "encoding/binary" "fmt" "io" "math" @@ -16,7 +17,7 @@ type frameHeader struct { WindowSize uint32 SingleSegment bool Checksum bool - DictID uint32 // Not stored. + DictID uint32 } const maxHeaderSize = 14 @@ -30,6 +31,24 @@ func (f frameHeader) appendTo(dst []byte) ([]byte, error) { if f.SingleSegment { fhd |= 1 << 5 } + + var dictIDContent []byte + if f.DictID > 0 { + var tmp [4]byte + if f.DictID < 256 { + fhd |= 1 + tmp[0] = uint8(f.DictID) + dictIDContent = tmp[:1] + } else if f.DictID < 1<<16 { + fhd |= 2 + binary.LittleEndian.PutUint16(tmp[:2], uint16(f.DictID)) + dictIDContent = tmp[:2] + } else { + fhd |= 3 + binary.LittleEndian.PutUint32(tmp[:4], f.DictID) + dictIDContent = tmp[:4] + } + } var fcs uint8 if f.ContentSize >= 256 { fcs++ @@ -40,6 +59,7 @@ func (f frameHeader) appendTo(dst []byte) ([]byte, error) { if f.ContentSize >= 0xffffffff { fcs++ } + fhd |= fcs << 6 dst = append(dst, fhd) @@ -48,7 +68,9 @@ func (f frameHeader) appendTo(dst []byte) ([]byte, error) { windowLog := (bits.Len32(f.WindowSize-1) - winLogMin) << 3 dst = append(dst, uint8(windowLog)) } - + if f.DictID > 0 { + dst = append(dst, dictIDContent...) + } switch fcs { case 0: if f.SingleSegment { diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go index f418f50fcd665..f783e32d251b4 100644 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -37,7 +37,7 @@ func (h *history) reset() { } h.decoders = sequenceDecs{} if h.huffTree != nil { - if h.dict == nil || h.dict.litDec != h.huffTree { + if h.dict == nil || h.dict.litEnc != h.huffTree { huffDecoderPool.Put(h.huffTree) } } @@ -55,7 +55,7 @@ func (h *history) setDict(dict *dict) { h.decoders.offsets = dict.ofDec h.decoders.matchLengths = dict.mlDec h.recentOffsets = dict.offsets - h.huffTree = dict.litDec + h.huffTree = dict.litEnc } // append bytes to history. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 7ff870400d508..b5c8ef133296a 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -196,6 +196,10 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { s.literals = s.literals[ll:] out := s.out + if mo == 0 && ml > 0 { + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + } + if mo > len(s.out)+len(hist) || mo > s.windowSize { if len(s.dict) == 0 { return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) @@ -218,10 +222,6 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { } } - if mo == 0 && ml > 0 { - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - } - // Copy from history. // TODO: Blocks without history could be made to ignore this completely. if v := mo - len(s.out); v > 0 { diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index 690428cd24015..841fd95acce98 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -111,7 +111,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { // Add empty last block r.block.reset(nil) r.block.last = true - err := r.block.encodeLits(false) + err := r.block.encodeLits(r.block.literals, false) if err != nil { return written, err } @@ -178,7 +178,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { r.err = ErrSnappyCorrupt return written, r.err } - err = r.block.encode(false, false) + err = r.block.encode(nil, false, false) switch err { case errIncompressible: r.block.popOffsets() @@ -188,7 +188,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { println("snappy.Decode:", err) return written, err } - err = r.block.encodeLits(false) + err = r.block.encodeLits(r.block.literals, false) if err != nil { return written, err } @@ -235,7 +235,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { r.err = ErrSnappyCorrupt return written, r.err } - err := r.block.encodeLits(false) + err := r.block.encodeLits(r.block.literals, false) if err != nil { return written, err } diff --git a/vendor/github.com/klauspost/pgzip/.travis.yml b/vendor/github.com/klauspost/pgzip/.travis.yml index 6e9fca0bac3a0..acfec4bb09d6e 100644 --- a/vendor/github.com/klauspost/pgzip/.travis.yml +++ b/vendor/github.com/klauspost/pgzip/.travis.yml @@ -1,19 +1,22 @@ language: go -sudo: false - os: - linux - osx go: - - 1.9.x - - 1.10.x + - 1.13.x + - 1.14.x + - 1.15.x - master -script: - - go test -v -cpu=1,2,4 . - - go test -v -cpu=2 -race -short . +env: + - GO111MODULE=off + +script: + - diff <(gofmt -d .) <(printf "") + - go test -v -cpu=1,2,4 . + - go test -v -cpu=2 -race -short . matrix: allow_failures: diff --git a/vendor/github.com/klauspost/pgzip/gunzip.go b/vendor/github.com/klauspost/pgzip/gunzip.go index 93efec714894d..d1ae730b25ff3 100644 --- a/vendor/github.com/klauspost/pgzip/gunzip.go +++ b/vendor/github.com/klauspost/pgzip/gunzip.go @@ -331,6 +331,16 @@ func (z *Reader) killReadAhead() error { // Wait for decompressor to be closed and return error, if any. e, ok := <-z.closeErr z.activeRA = false + + for blk := range z.readAhead { + if blk.b != nil { + z.blockPool <- blk.b + } + } + if cap(z.current) > 0 { + z.blockPool <- z.current + z.current = nil + } if !ok { // Channel is closed, so if there was any error it has already been returned. return nil @@ -418,6 +428,7 @@ func (z *Reader) doReadAhead() { case z.readAhead <- read{b: buf, err: err}: case <-closeReader: // Sent on close, we don't care about the next results + z.blockPool <- buf return } if err != nil { diff --git a/vendor/github.com/mholt/archiver/v3/README.md b/vendor/github.com/mholt/archiver/v3/README.md index 5f7cd514ffab0..c8de5e7e551b7 100644 --- a/vendor/github.com/mholt/archiver/v3/README.md +++ b/vendor/github.com/mholt/archiver/v3/README.md @@ -1,4 +1,4 @@ -# archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/mholt/archiver?tab=doc) +# archiver [![archiver GoDoc](https://img.shields.io/badge/reference-godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/mholt/archiver?tab=doc) [![Ubuntu-latest](https://github.com/mholt/archiver/actions/workflows/ubuntu-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/ubuntu-latest.yml) [![Macos-latest](https://github.com/mholt/archiver/actions/workflows/macos-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/macos-latest.yml) [![Windows-latest](https://github.com/mholt/archiver/actions/workflows/windows-latest.yml/badge.svg)](https://github.com/mholt/archiver/actions/workflows/windows-latest.yml) Introducing **Archiver 3.1** - a cross-platform, multi-format archive utility and Go library. A powerful and flexible library meets an elegant CLI in this generic replacement for several platform-specific or format-specific archive utilities. @@ -75,7 +75,7 @@ curl.exe -fsS -A MS https://webinstall.dev/arc | powershell To install the runnable binary to your \$GOPATH/bin: ```bash -go get github.com/mholt/archiver/cmd/arc +go install github.com/mholt/archiver/v3/cmd/arc@latest ``` ### Manually diff --git a/vendor/github.com/mholt/archiver/v3/SECURITY.md b/vendor/github.com/mholt/archiver/v3/SECURITY.md new file mode 100644 index 0000000000000..f915712462e5a --- /dev/null +++ b/vendor/github.com/mholt/archiver/v3/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| >= 3.x | :white_check_mark: | +| < 3.0 | :x: | + +## Reporting a Vulnerability + +Please send the details to both of us: + +- AJ ONeal +- Matthew Holt diff --git a/vendor/github.com/mholt/archiver/v3/archiver.go b/vendor/github.com/mholt/archiver/v3/archiver.go index b09699d676a48..6fdadadc4c2d0 100644 --- a/vendor/github.com/mholt/archiver/v3/archiver.go +++ b/vendor/github.com/mholt/archiver/v3/archiver.go @@ -125,6 +125,9 @@ type File struct { type FileInfo struct { os.FileInfo CustomName string + // Stores path to the source. + // Used when reading a symlink. + SourcePath string } // Name returns fi.CustomName if not empty; diff --git a/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml b/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml deleted file mode 100644 index 672dce356c2f3..0000000000000 --- a/vendor/github.com/mholt/archiver/v3/azure-pipelines.yml +++ /dev/null @@ -1,111 +0,0 @@ -trigger: -- master - -strategy: - matrix: - linux: - imageName: ubuntu-18.04 - gorootDir: /usr/local - mac: - # Mojave - imageName: macos-10.14 - gorootDir: /usr/local - windows: - imageName: windows-2019 - gorootDir: C:\ - -pool: - vmImage: $(imageName) - -variables: - GOROOT: $(gorootDir)/go - GOPATH: $(system.defaultWorkingDirectory)/gopath - GOBIN: $(GOPATH)/bin - modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' - # TODO: Enable modules after upstream dependency zstd supports them - # TODO: modules should be the default in Go 1.13, so this won't be needed - #GO111MODULE: on - -steps: - -- bash: git config --global core.autocrlf false - displayName: "Disable line ending conversion for git to" - -- checkout: self - -- bash: | - latestGo=$(curl "https://golang.org/VERSION?m=text") - echo "##vso[task.setvariable variable=LATEST_GO]$latestGo" - echo "Latest Go version: $latestGo" - displayName: "Get latest Go version" - -- bash: | - sudo rm -f $(which go) - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - mkdir -p '$(modulePath)' - shopt -s extglob - shopt -s dotglob - mv !(gopath) '$(modulePath)' - displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH - -# Install Go (this varies by platform) - -- bash: | - wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz" - sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz" - condition: eq( variables['Agent.OS'], 'Linux' ) - displayName: Install Go on Linux - -- bash: | - wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz" - sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz" - condition: eq( variables['Agent.OS'], 'Darwin' ) - displayName: Install Go on macOS - -- powershell: | - echo "Downloading Go..." - # Windows comes with BSD curl, which is MUCH faster than the native Windows HTTP - curl.exe -fsSL -o "$(LATEST_GO).windows-amd64.zip" "https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip" - echo "Extracting Go..." - # Yes, Windows tar (BSD tar) handles zip files. Who knew!? - # (and it's MUCH, MUCH, MUCH faster than the native Windows expander) - tar.exe xf "$(LATEST_GO).windows-amd64.zip" -C "$(gorootDir)" - condition: eq( variables['Agent.OS'], 'Windows_NT' ) - displayName: Install Go on Windows - -- bash: | - printf "Using go at: $(which go)\n" - printf "Go version: $(go version)\n" - printf "\n\nGo environment:\n\n" - go env - printf "\n\nSystem environment:\n\n" - env - displayName: Print Go version and environment - -- script: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.31.0 - ./bin/golangci-lint run -E gofmt -E goimports -E misspell ./... - workingDirectory: '$(modulePath)' - condition: eq( variables['Agent.OS'], 'Linux' ) - displayName: Run Lint - -- bash: | - go mod tidy - if [ ! -z "$(git status --porcelain go.mod)" ]; then - printf "go.mod has modifications\n" - git diff go.mod - exit 1 - fi - if [ ! -z "$(git status --porcelain go.sum)" ]; then - printf "go.sum has modifications\n" - git diff go.sum - exit 1 - fi - workingDirectory: '$(modulePath)' - displayName: Ensure that module definition and checksums are correct - -- script: | - go test -race ./... - workingDirectory: '$(modulePath)' - displayName: Run tests diff --git a/vendor/github.com/mholt/archiver/v3/go.mod b/vendor/github.com/mholt/archiver/v3/go.mod index 8d8e4e16fa5a3..ca511dc9af914 100644 --- a/vendor/github.com/mholt/archiver/v3/go.mod +++ b/vendor/github.com/mholt/archiver/v3/go.mod @@ -3,13 +3,13 @@ module github.com/mholt/archiver/v3 go 1.13 require ( - github.com/andybalholm/brotli v1.0.0 - github.com/dsnet/compress v0.0.1 - github.com/golang/snappy v0.0.1 - github.com/klauspost/compress v1.10.10 - github.com/klauspost/pgzip v1.2.4 + github.com/andybalholm/brotli v1.0.1 + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 + github.com/golang/snappy v0.0.2 + github.com/klauspost/compress v1.11.4 + github.com/klauspost/pgzip v1.2.5 github.com/nwaples/rardecode v1.1.0 - github.com/pierrec/lz4/v4 v4.0.3 - github.com/ulikunitz/xz v0.5.7 + github.com/pierrec/lz4/v4 v4.1.2 + github.com/ulikunitz/xz v0.5.9 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 ) diff --git a/vendor/github.com/mholt/archiver/v3/go.sum b/vendor/github.com/mholt/archiver/v3/go.sum index adf9ee7dbb0c5..e874a9ce40332 100644 --- a/vendor/github.com/mholt/archiver/v3/go.sum +++ b/vendor/github.com/mholt/archiver/v3/go.sum @@ -1,24 +1,25 @@ -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= -github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= -github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/github.com/mholt/archiver/v3/rar.go b/vendor/github.com/mholt/archiver/v3/rar.go index 56c2a3e3e8397..35fd60b676b51 100644 --- a/vendor/github.com/mholt/archiver/v3/rar.go +++ b/vendor/github.com/mholt/archiver/v3/rar.go @@ -405,9 +405,9 @@ func (*Rar) Match(file io.ReadSeeker) (bool, error) { if n, err := file.Read(buf); err != nil || n < 8 { return false, nil } - hasTarHeader := bytes.Equal(buf[:7], []byte("Rar!\x1a\x07\x00")) || // ver 1.5 + hasRarHeader := bytes.Equal(buf[:7], []byte("Rar!\x1a\x07\x00")) || // ver 1.5 bytes.Equal(buf, []byte("Rar!\x1a\x07\x01\x00")) // ver 5.0 - return hasTarHeader, nil + return hasRarHeader, nil } func (r *Rar) String() string { return "rar" } diff --git a/vendor/github.com/mholt/archiver/v3/tar.go b/vendor/github.com/mholt/archiver/v3/tar.go index 60c58fd1358ba..be8986657dcc5 100644 --- a/vendor/github.com/mholt/archiver/v3/tar.go +++ b/vendor/github.com/mholt/archiver/v3/tar.go @@ -327,6 +327,7 @@ func (t *Tar) writeWalk(source, topLevelFolder, destination string) error { FileInfo: FileInfo{ FileInfo: info, CustomName: nameInArchive, + SourcePath: fpath, }, ReadCloser: file, }) @@ -372,10 +373,14 @@ func (t *Tar) Write(f File) error { var linkTarget string if isSymlink(f) { + fi, ok := f.FileInfo.(FileInfo) + if !ok { + return fmt.Errorf("failed to cast fs.FileInfo to archiver.FileInfo: %v", f) + } var err error - linkTarget, err = os.Readlink(f.Name()) + linkTarget, err = os.Readlink(fi.SourcePath) if err != nil { - return fmt.Errorf("%s: readlink: %v", f.Name(), err) + return fmt.Errorf("%s: readlink: %v", fi.SourcePath, err) } } @@ -386,7 +391,7 @@ func (t *Tar) Write(f File) error { err = t.tw.WriteHeader(hdr) if err != nil { - return fmt.Errorf("%s: writing header: %v", hdr.Name, err) + return fmt.Errorf("%s: writing header: %w", hdr.Name, err) } if f.IsDir() { @@ -399,7 +404,7 @@ func (t *Tar) Write(f File) error { } _, err := io.Copy(t.tw, f) if err != nil { - return fmt.Errorf("%s: copying contents: %v", f.Name(), err) + return fmt.Errorf("%s: copying contents: %w", f.Name(), err) } } diff --git a/vendor/github.com/mholt/archiver/v3/zip.go b/vendor/github.com/mholt/archiver/v3/zip.go index 81694bdc45e1f..c6af8efbc6727 100644 --- a/vendor/github.com/mholt/archiver/v3/zip.go +++ b/vendor/github.com/mholt/archiver/v3/zip.go @@ -1,7 +1,6 @@ package archiver import ( - "archive/zip" "bytes" "compress/flate" "fmt" @@ -14,6 +13,7 @@ import ( "strings" "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zip" "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" ) @@ -350,6 +350,7 @@ func (z *Zip) writeWalk(source, topLevelFolder, destination string) error { FileInfo: FileInfo{ FileInfo: info, CustomName: nameInArchive, + SourcePath: fpath, }, ReadCloser: file, }) @@ -420,7 +421,7 @@ func (z *Zip) Write(f File) error { writer, err := z.zw.CreateHeader(header) if err != nil { - return fmt.Errorf("%s: making header: %v", f.Name(), err) + return fmt.Errorf("%s: making header: %w", f.Name(), err) } return z.writeFile(f, writer) @@ -431,14 +432,18 @@ func (z *Zip) writeFile(f File, writer io.Writer) error { return nil // directories have no contents } if isSymlink(f) { + fi, ok := f.FileInfo.(FileInfo) + if !ok { + return fmt.Errorf("failed to cast fs.FileInfo to archiver.FileInfo: %v", f) + } // file body for symlinks is the symlink target - linkTarget, err := os.Readlink(f.Name()) + linkTarget, err := os.Readlink(fi.SourcePath) if err != nil { - return fmt.Errorf("%s: readlink: %v", f.Name(), err) + return fmt.Errorf("%s: readlink: %v", fi.SourcePath, err) } _, err = writer.Write([]byte(filepath.ToSlash(linkTarget))) if err != nil { - return fmt.Errorf("%s: writing symlink target: %v", f.Name(), err) + return fmt.Errorf("%s: writing symlink target: %v", fi.SourcePath, err) } return nil } @@ -448,7 +453,7 @@ func (z *Zip) writeFile(f File, writer io.Writer) error { } _, err := io.Copy(writer, f) if err != nil { - return fmt.Errorf("%s: copying contents: %v", f.Name(), err) + return fmt.Errorf("%s: copying contents: %w", f.Name(), err) } return nil diff --git a/vendor/github.com/pierrec/lz4/v4/.gitignore b/vendor/github.com/pierrec/lz4/v4/.gitignore index 5e987350471d0..5d7e88de0a3d5 100644 --- a/vendor/github.com/pierrec/lz4/v4/.gitignore +++ b/vendor/github.com/pierrec/lz4/v4/.gitignore @@ -31,4 +31,6 @@ Temporary Items # End of https://www.gitignore.io/api/macos cmd/*/*exe -.idea \ No newline at end of file +.idea + +fuzz/*.zip diff --git a/vendor/github.com/pierrec/lz4/v4/.travis.yml b/vendor/github.com/pierrec/lz4/v4/.travis.yml deleted file mode 100644 index 4a9819e03a130..0000000000000 --- a/vendor/github.com/pierrec/lz4/v4/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: go - -env: - - GO111MODULE=off - -go: - - 1.13.x - - 1.14.x - -matrix: - fast_finish: true - -sudo: false - -script: - - go test -v -cpu=2 - - go test -v -cpu=2 -race - - go test -v -cpu=2 -tags noasm - - go test -v -cpu=2 -race -tags noasm diff --git a/vendor/github.com/pierrec/lz4/v4/README.md b/vendor/github.com/pierrec/lz4/v4/README.md index 4ee388e81bfb9..df027e2c30189 100644 --- a/vendor/github.com/pierrec/lz4/v4/README.md +++ b/vendor/github.com/pierrec/lz4/v4/README.md @@ -1,7 +1,7 @@ # lz4 : LZ4 compression in pure Go -[![GoDoc](https://godoc.org/github.com/pierrec/lz4?status.svg)](https://godoc.org/github.com/pierrec/lz4) -[![Build Status](https://travis-ci.org/pierrec/lz4.svg?branch=master)](https://travis-ci.org/pierrec/lz4) +[![Go Reference](https://pkg.go.dev/badge/github.com/pierrec/lz4/v4.svg)](https://pkg.go.dev/github.com/pierrec/lz4/v4) +[![CI](https://github.com/pierrec/lz4/workflows/ci/badge.svg)](https://github.com/pierrec/lz4/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4) [![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags) @@ -15,13 +15,13 @@ The implementation is based on the reference C [one](https://github.com/lz4/lz4) Assuming you have the go toolchain installed: ``` -go get github.com/pierrec/lz4 +go get github.com/pierrec/lz4/v4 ``` There is a command line interface tool to compress and decompress LZ4 files. ``` -go install github.com/pierrec/lz4/cmd/lz4c +go install github.com/pierrec/lz4/v4/cmd/lz4c ``` Usage diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go index e634c2cb8aa77..f38264943084f 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go @@ -132,7 +132,7 @@ func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { offset := si - ref - if ref < 0 || uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { + if offset <= 0 || offset >= winSize || uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { // No match. Start calculating another hash. // The processor can usually do this out-of-order. h = blockHash(match >> 16) @@ -142,13 +142,13 @@ func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { si += 1 offset = si - ref2 - if ref2 < 0 || uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) { + if offset <= 0 || offset >= winSize || uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) { // No match. Check the third match at si+2 si += 1 offset = si - ref3 c.put(h, si) - if ref3 < 0 || uint32(match>>16) != binary.LittleEndian.Uint32(src[ref3:]) { + if offset <= 0 || offset >= winSize || uint32(match>>16) != binary.LittleEndian.Uint32(src[ref3:]) { // Skip one extra byte (at si+3) before we check 3 matches again. si += 2 + (si-anchor)>>adaptSkipLog continue diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go index 098a5cf9b968e..e6cf88d71c03a 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go @@ -8,6 +8,8 @@ const ( Block256Kb Block1Mb Block4Mb + Block8Mb = 2 * Block4Mb + legacyBlockSize = Block8Mb + Block8Mb/255 + 16 // CompressBound(Block8Mb) ) var ( @@ -15,6 +17,7 @@ var ( BlockPool256K = sync.Pool{New: func() interface{} { return make([]byte, Block256Kb) }} BlockPool1M = sync.Pool{New: func() interface{} { return make([]byte, Block1Mb) }} BlockPool4M = sync.Pool{New: func() interface{} { return make([]byte, Block4Mb) }} + BlockPool8M = sync.Pool{New: func() interface{} { return make([]byte, legacyBlockSize) }} ) func Index(b uint32) BlockSizeIndex { @@ -27,6 +30,8 @@ func Index(b uint32) BlockSizeIndex { return 6 case Block4Mb: return 7 + case Block8Mb: // only valid in legacy mode + return 3 } return 0 } @@ -56,29 +61,25 @@ func (b BlockSizeIndex) Get() []byte { buf = BlockPool1M.Get() case 7: buf = BlockPool4M.Get() + case 3: + buf = BlockPool8M.Get() } return buf.([]byte) } -func (b BlockSizeIndex) Put(buf []byte) { +func Put(buf []byte) { // Safeguard: do not allow invalid buffers. - switch c := uint32(cap(buf)); b { - case 4: - if c == Block64Kb { - BlockPool64K.Put(buf[:c]) - } - case 5: - if c == Block256Kb { - BlockPool256K.Put(buf[:c]) - } - case 6: - if c == Block1Mb { - BlockPool1M.Put(buf[:c]) - } - case 7: - if c == Block4Mb { - BlockPool4M.Put(buf[:c]) - } + switch c := cap(buf); uint32(c) { + case Block64Kb: + BlockPool64K.Put(buf[:c]) + case Block256Kb: + BlockPool256K.Put(buf[:c]) + case Block1Mb: + BlockPool1M.Put(buf[:c]) + case Block4Mb: + BlockPool4M.Put(buf[:c]) + case legacyBlockSize: + BlockPool8M.Put(buf[:c]) } } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s index ec94b7b3c3162..64be9adcaa871 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s @@ -140,25 +140,22 @@ readMatchlenLoop: BEQ readMatchlenLoop readMatchlenDone: - ADD minMatch, len - - // Bounds check dst+len and match = dst-offset. + // Bounds check dst+len+minMatch and match = dst-offset. ADD dst, len, tmp1 + ADD minMatch, tmp1 CMP dstend, tmp1 //BHI shortDst // Uncomment for distinct error codes. SUB offset, dst, match CMP.LS match, dstorig BHI corrupt - // If the offset is at least four (len is, because of minMatch), - // do a four-way unrolled byte copy loop. Using MOVD instead of four - // byte loads is much faster, but to remain portable we'd have to - // align match first, which in turn is too expensive. - CMP $4, offset - BLO copyMatch - - SUB $4, len + // Since len+minMatch is at least four, we can do a 4× unrolled + // byte copy loop. Using MOVW instead of four byte loads is faster, + // but to remain portable we'd have to align match first, which is + // too expensive. By alternating loads and stores, we also handle + // the case offset < 4. copyMatch4: + SUB.S $4, len MOVBU.P 4(match), tmp1 MOVB.P tmp1, 4(dst) MOVBU -3(match), tmp2 @@ -167,7 +164,6 @@ copyMatch4: MOVB tmp3, -2(dst) MOVBU -1(match), tmp1 MOVB tmp1, -1(dst) - SUB.S $4, len BPL copyMatch4 // Restore len, which is now negative. @@ -175,7 +171,7 @@ copyMatch4: BEQ copyMatchDone copyMatch: - // Simple byte-at-a-time copy. + // Finish with a byte-at-a-time copy. SUB.S $1, len MOVBU.P 1(match), tmp2 MOVB.P tmp2, 1(dst) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go index 9065653a9f28d..52df2f2b8ec25 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go @@ -2,7 +2,13 @@ package lz4block +import "encoding/binary" + func decodeBlock(dst, src []byte) (ret int) { + // Restrict capacities so we don't read or write out of bounds. + dst = dst[:len(dst):len(dst)] + src = src[:len(src):len(src)] + const hasError = -2 defer func() { if recover() != nil { @@ -32,7 +38,7 @@ func decodeBlock(dst, src []byte) (ret int) { // if the match length (4..18) fits within the literals, then copy // all 18 bytes, even if not all are part of the literals. mLen += 4 - if offset := uint(src[si]) | uint(src[si+1])<<8; mLen <= offset { + if offset := u16(src[si:]); mLen <= offset { i := di - offset end := i + 18 if end > uint(len(dst)) { @@ -66,7 +72,7 @@ func decodeBlock(dst, src []byte) (ret int) { return hasError } - offset := uint(src[si]) | uint(src[si+1])<<8 + offset := u16(src[si:]) if offset == 0 { return hasError } @@ -98,3 +104,5 @@ func decodeBlock(dst, src []byte) (ret int) { di += uint(copy(dst[di:di+mLen], expanded[:mLen])) } } + +func u16(p []byte) uint { return uint(binary.LittleEndian.Uint16(p)) } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go new file mode 100644 index 0000000000000..5e0c062ec0b81 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go @@ -0,0 +1,332 @@ +package lz4stream + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/xxh32" +) + +type Blocks struct { + Block *FrameDataBlock + Blocks chan chan *FrameDataBlock + mu sync.Mutex + err error +} + +func (b *Blocks) initW(f *Frame, dst io.Writer, num int) { + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return + } + b.Block = nil + if cap(b.Blocks) != num { + b.Blocks = make(chan chan *FrameDataBlock, num) + } + // goroutine managing concurrent block compression goroutines. + go func() { + // Process next block compression item. + for c := range b.Blocks { + // Read the next compressed block result. + // Waiting here ensures that the blocks are output in the order they were sent. + // The incoming channel is always closed as it indicates to the caller that + // the block has been processed. + block := <-c + if block == nil { + // Notify the block compression routine that we are done with its result. + // This is used when a sentinel block is sent to terminate the compression. + close(c) + return + } + // Do not attempt to write the block upon any previous failure. + if b.err == nil { + // Write the block. + if err := block.Write(f, dst); err != nil { + // Keep the first error. + b.err = err + // All pending compression goroutines need to shut down, so we need to keep going. + } + } + close(c) + } + }() +} + +func (b *Blocks) close(f *Frame, num int) error { + if num == 1 { + if b.Block != nil { + b.Block.Close(f) + } + err := b.err + b.err = nil + return err + } + if b.Blocks == nil { + err := b.err + b.err = nil + return err + } + c := make(chan *FrameDataBlock) + b.Blocks <- c + c <- nil + <-c + err := b.err + b.err = nil + return err +} + +// ErrorR returns any error set while uncompressing a stream. +func (b *Blocks) ErrorR() error { + b.mu.Lock() + defer b.mu.Unlock() + return b.err +} + +// initR returns a channel that streams the uncompressed blocks if in concurrent +// mode and no error. When the channel is closed, check for any error with b.ErrorR. +// +// If not in concurrent mode, the uncompressed block is b.Block and the returned error +// needs to be checked. +func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { + size := f.Descriptor.Flags.BlockSizeIndex() + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return nil, nil + } + b.Block = nil + blocks := make(chan chan []byte, num) + // data receives the uncompressed blocks. + data := make(chan []byte) + // Read blocks from the source sequentially + // and uncompress them concurrently. + + // In legacy mode, accrue the uncompress sizes in cum. + var cum uint32 + go func() { + var cumx uint32 + var err error + for b.ErrorR() == nil { + block := NewFrameDataBlock(f) + cumx, err = block.Read(f, src, 0) + if err != nil { + break + } + // Recheck for an error as reading may be slow and uncompressing is expensive. + if b.ErrorR() != nil { + break + } + c := make(chan []byte) + blocks <- c + go func() { + data, err := block.Uncompress(f, size.Get(), false) + if err != nil { + b.closeR(err) + } else { + c <- data + } + }() + } + // End the collection loop and the data channel. + c := make(chan []byte) + blocks <- c + c <- nil // signal the collection loop that we are done + <-c // wait for the collect loop to complete + if f.isLegacy() && cum == cumx { + err = io.EOF + } + b.closeR(err) + close(data) + }() + // Collect the uncompressed blocks and make them available + // on the returned channel. + go func(leg bool) { + defer close(blocks) + for c := range blocks { + buf := <-c + if buf == nil { + // Signal to end the loop. + close(c) + return + } + // Perform checksum now as the blocks are received in order. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(buf) + } + if leg { + cum += uint32(len(buf)) + } + data <- buf + close(c) + } + }(f.isLegacy()) + return data, nil +} + +// closeR safely sets the error on b if not already set. +func (b *Blocks) closeR(err error) { + b.mu.Lock() + if b.err == nil { + b.err = err + } + b.mu.Unlock() +} + +func NewFrameDataBlock(f *Frame) *FrameDataBlock { + buf := f.Descriptor.Flags.BlockSizeIndex().Get() + return &FrameDataBlock{Data: buf, data: buf} +} + +type FrameDataBlock struct { + Size DataBlockSize + Data []byte // compressed or uncompressed data (.data or .src) + Checksum uint32 + data []byte // buffer for compressed data + src []byte // uncompressed data + err error // used in concurrent mode +} + +func (b *FrameDataBlock) Close(f *Frame) { + b.Size = 0 + b.Checksum = 0 + b.err = nil + if b.data != nil { + // Block was not already closed. + lz4block.Put(b.data) + b.Data = nil + b.data = nil + b.src = nil + } +} + +// Block compression errors are ignored since the buffer is sized appropriately. +func (b *FrameDataBlock) Compress(f *Frame, src []byte, level lz4block.CompressionLevel) *FrameDataBlock { + data := b.data + if f.isLegacy() { + data = data[:cap(data)] + } else { + data = data[:len(src)] // trigger the incompressible flag in CompressBlock + } + var n int + switch level { + case lz4block.Fast: + n, _ = lz4block.CompressBlock(src, data) + default: + n, _ = lz4block.CompressBlockHC(src, data, level) + } + if n == 0 { + b.Size.UncompressedSet(true) + b.Data = src + } else { + b.Size.UncompressedSet(false) + b.Data = data[:n] + } + b.Size.sizeSet(len(b.Data)) + b.src = src // keep track of the source for content checksum + + if f.Descriptor.Flags.BlockChecksum() { + b.Checksum = xxh32.ChecksumZero(src) + } + return b +} + +func (b *FrameDataBlock) Write(f *Frame, dst io.Writer) error { + // Write is called in the same order as blocks are compressed, + // so content checksum must be done here. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(b.src) + } + buf := f.buf[:] + binary.LittleEndian.PutUint32(buf, uint32(b.Size)) + if _, err := dst.Write(buf[:4]); err != nil { + return err + } + + if _, err := dst.Write(b.Data); err != nil { + return err + } + + if b.Checksum == 0 { + return nil + } + binary.LittleEndian.PutUint32(buf, b.Checksum) + _, err := dst.Write(buf[:4]) + return err +} + +// Read updates b with the next block data, size and checksum if available. +func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, error) { + x, err := f.readUint32(src) + if err != nil { + return 0, err + } + if f.isLegacy() { + switch x { + case frameMagicLegacy: + // Concatenated legacy frame. + return b.Read(f, src, cum) + case cum: + // Only works in non concurrent mode, for concurrent mode + // it is handled separately. + // Linux kernel format appends the total uncompressed size at the end. + return 0, io.EOF + } + } else if x == 0 { + // Marker for end of stream. + return 0, io.EOF + } + b.Size = DataBlockSize(x) + + size := b.Size.size() + if size > cap(b.data) { + return x, lz4errors.ErrOptionInvalidBlockSize + } + b.data = b.data[:size] + if _, err := io.ReadFull(src, b.data); err != nil { + return x, err + } + if f.Descriptor.Flags.BlockChecksum() { + sum, err := f.readUint32(src) + if err != nil { + return 0, err + } + b.Checksum = sum + } + return x, nil +} + +func (b *FrameDataBlock) Uncompress(f *Frame, dst []byte, sum bool) ([]byte, error) { + if b.Size.Uncompressed() { + n := copy(dst, b.data) + dst = dst[:n] + } else { + n, err := lz4block.UncompressBlock(b.data, dst) + if err != nil { + return nil, err + } + dst = dst[:n] + } + if f.Descriptor.Flags.BlockChecksum() { + if c := xxh32.ChecksumZero(dst); c != b.Checksum { + err := fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidBlockChecksum, c, b.Checksum) + return nil, err + } + } + if sum && f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(dst) + } + return dst, nil +} + +func (f *Frame) readUint32(r io.Reader) (x uint32, err error) { + if _, err = io.ReadFull(r, f.buf[:4]); err != nil { + return + } + x = binary.LittleEndian.Uint32(f.buf[:4]) + return +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go index f6ab4decb7b01..cfbd5674d9d10 100644 --- a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go @@ -15,8 +15,9 @@ import ( //go:generate go run gen.go const ( - frameMagic uint32 = 0x184D2204 - frameSkipMagic uint32 = 0x184D2A50 + frameMagic uint32 = 0x184D2204 + frameSkipMagic uint32 = 0x184D2A50 + frameMagicLegacy uint32 = 0x184C2102 ) func NewFrame() *Frame { @@ -38,21 +39,30 @@ func (f *Frame) Reset(num int) { f.Magic = 0 f.Descriptor.Checksum = 0 f.Descriptor.ContentSize = 0 - _ = f.Blocks.closeW(f, num) + _ = f.Blocks.close(f, num) f.Checksum = 0 } -func (f *Frame) InitW(dst io.Writer, num int) { - f.Magic = frameMagic - f.Descriptor.initW() +func (f *Frame) InitW(dst io.Writer, num int, legacy bool) { + if legacy { + f.Magic = frameMagicLegacy + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + } else { + f.Magic = frameMagic + f.Descriptor.initW() + } f.Blocks.initW(f, dst, num) f.checksum.Reset() } func (f *Frame) CloseW(dst io.Writer, num int) error { - if err := f.Blocks.closeW(f, num); err != nil { + if err := f.Blocks.close(f, num); err != nil { return err } + if f.isLegacy() { + return nil + } buf := f.buf[:0] // End mark (data block size of uint32(0)). buf = append(buf, 0, 0, 0, 0) @@ -63,41 +73,47 @@ func (f *Frame) CloseW(dst io.Writer, num int) error { return err } -func (f *Frame) InitR(src io.Reader) error { +func (f *Frame) isLegacy() bool { + return f.Magic == frameMagicLegacy +} + +func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { if f.Magic > 0 { // Header already read. - return nil + return nil, nil } newFrame: var err error if f.Magic, err = f.readUint32(src); err != nil { - return err + return nil, err } switch m := f.Magic; { - case m == frameMagic: + case m == frameMagic || m == frameMagicLegacy: // All 16 values of frameSkipMagic are valid. case m>>8 == frameSkipMagic>>8: - var skip uint32 - if err := binary.Read(src, binary.LittleEndian, &skip); err != nil { - return err + skip, err := f.readUint32(src) + if err != nil { + return nil, err } if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { - return err + return nil, err } goto newFrame default: - return lz4errors.ErrInvalidFrame + return nil, lz4errors.ErrInvalidFrame } if err := f.Descriptor.initR(f, src); err != nil { - return err + return nil, err } - f.Blocks.initR(f) f.checksum.Reset() - return nil + return f.Blocks.initR(f, num, src) } func (f *Frame) CloseR(src io.Reader) (err error) { + if f.isLegacy() { + return nil + } if !f.Descriptor.Flags.ContentChecksum() { return nil } @@ -127,23 +143,31 @@ func (fd *FrameDescriptor) Write(f *Frame, dst io.Writer) error { return nil } - buf := f.buf[:4+2] + buf := f.buf[:4] // Write the magic number here even though it belongs to the Frame. binary.LittleEndian.PutUint32(buf, f.Magic) - binary.LittleEndian.PutUint16(buf[4:], uint16(fd.Flags)) + if !f.isLegacy() { + buf = buf[:4+2] + binary.LittleEndian.PutUint16(buf[4:], uint16(fd.Flags)) - if fd.Flags.Size() { - buf = buf[:4+2+8] - binary.LittleEndian.PutUint64(buf[4+2:], fd.ContentSize) + if fd.Flags.Size() { + buf = buf[:4+2+8] + binary.LittleEndian.PutUint64(buf[4+2:], fd.ContentSize) + } + fd.Checksum = descriptorChecksum(buf[4:]) + buf = append(buf, fd.Checksum) } - fd.Checksum = descriptorChecksum(buf[4:]) - buf = append(buf, fd.Checksum) _, err := dst.Write(buf) return err } func (fd *FrameDescriptor) initR(f *Frame, src io.Reader) error { + if f.isLegacy() { + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + return nil + } // Read the flags and the checksum, hoping that there is not content size. buf := f.buf[:3] if _, err := io.ReadFull(src, buf); err != nil { @@ -174,204 +198,3 @@ func (fd *FrameDescriptor) initR(f *Frame, src io.Reader) error { func descriptorChecksum(buf []byte) byte { return byte(xxh32.ChecksumZero(buf) >> 8) } - -type Blocks struct { - Block *FrameDataBlock - Blocks chan chan *FrameDataBlock - err error -} - -func (b *Blocks) initW(f *Frame, dst io.Writer, num int) { - size := f.Descriptor.Flags.BlockSizeIndex() - if num == 1 { - b.Blocks = nil - b.Block = NewFrameDataBlock(size) - return - } - b.Block = nil - if cap(b.Blocks) != num { - b.Blocks = make(chan chan *FrameDataBlock, num) - } - // goroutine managing concurrent block compression goroutines. - go func() { - // Process next block compression item. - for c := range b.Blocks { - // Read the next compressed block result. - // Waiting here ensures that the blocks are output in the order they were sent. - // The incoming channel is always closed as it indicates to the caller that - // the block has been processed. - block := <-c - if block == nil { - // Notify the block compression routine that we are done with its result. - // This is used when a sentinel block is sent to terminate the compression. - close(c) - return - } - // Do not attempt to write the block upon any previous failure. - if b.err == nil { - // Write the block. - if err := block.Write(f, dst); err != nil && b.err == nil { - // Keep the first error. - b.err = err - // All pending compression goroutines need to shut down, so we need to keep going. - } - } - close(c) - } - }() -} - -func (b *Blocks) closeW(f *Frame, num int) error { - if num == 1 { - if b.Block == nil { - // Not initialized yet. - return nil - } - b.Block.CloseW(f) - return nil - } - if b.Blocks == nil { - // Not initialized yet. - return nil - } - c := make(chan *FrameDataBlock) - b.Blocks <- c - c <- nil - <-c - err := b.err - b.err = nil - return err -} - -func (b *Blocks) initR(f *Frame) { - size := f.Descriptor.Flags.BlockSizeIndex() - b.Block = NewFrameDataBlock(size) -} - -func NewFrameDataBlock(size lz4block.BlockSizeIndex) *FrameDataBlock { - buf := size.Get() - return &FrameDataBlock{Data: buf, data: buf} -} - -type FrameDataBlock struct { - Size DataBlockSize - Data []byte // compressed or uncompressed data (.data or .src) - Checksum uint32 - data []byte // buffer for compressed data - src []byte // uncompressed data -} - -func (b *FrameDataBlock) CloseW(f *Frame) { - if b.data != nil { - // Block was not already closed. - size := f.Descriptor.Flags.BlockSizeIndex() - size.Put(b.data) - b.Data = nil - b.data = nil - b.src = nil - } -} - -// Block compression errors are ignored since the buffer is sized appropriately. -func (b *FrameDataBlock) Compress(f *Frame, src []byte, level lz4block.CompressionLevel) *FrameDataBlock { - data := b.data[:len(src)] // trigger the incompressible flag in CompressBlock - var n int - switch level { - case lz4block.Fast: - n, _ = lz4block.CompressBlock(src, data) - default: - n, _ = lz4block.CompressBlockHC(src, data, level) - } - if n == 0 { - b.Size.UncompressedSet(true) - b.Data = src - } else { - b.Size.UncompressedSet(false) - b.Data = data[:n] - } - b.Size.sizeSet(len(b.Data)) - b.src = src // keep track of the source for content checksum - - if f.Descriptor.Flags.BlockChecksum() { - b.Checksum = xxh32.ChecksumZero(src) - } - return b -} - -func (b *FrameDataBlock) Write(f *Frame, dst io.Writer) error { - if f.Descriptor.Flags.ContentChecksum() { - _, _ = f.checksum.Write(b.src) - } - buf := f.buf[:] - binary.LittleEndian.PutUint32(buf, uint32(b.Size)) - if _, err := dst.Write(buf[:4]); err != nil { - return err - } - - if _, err := dst.Write(b.Data); err != nil { - return err - } - - if b.Checksum == 0 { - return nil - } - binary.LittleEndian.PutUint32(buf, b.Checksum) - _, err := dst.Write(buf[:4]) - return err -} - -func (b *FrameDataBlock) Uncompress(f *Frame, src io.Reader, dst []byte) (int, error) { - x, err := f.readUint32(src) - if err != nil { - return 0, err - } - b.Size = DataBlockSize(x) - if b.Size == 0 { - // End of frame reached. - return 0, io.EOF - } - - isCompressed := !b.Size.Uncompressed() - size := b.Size.size() - var data []byte - if isCompressed { - // Data is first copied into b.Data and then it will get uncompressed into dst. - data = b.Data - } else { - // Data is directly copied into dst as it is not compressed. - data = dst - } - data = data[:size] - if _, err := io.ReadFull(src, data); err != nil { - return 0, err - } - if isCompressed { - n, err := lz4block.UncompressBlock(data, dst) - if err != nil { - return 0, err - } - data = dst[:n] - } - - if f.Descriptor.Flags.BlockChecksum() { - var err error - if b.Checksum, err = f.readUint32(src); err != nil { - return 0, err - } - if c := xxh32.ChecksumZero(data); c != b.Checksum { - return 0, fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidBlockChecksum, c, b.Checksum) - } - } - if f.Descriptor.Flags.ContentChecksum() { - _, _ = f.checksum.Write(data) - } - return len(data), nil -} - -func (f *Frame) readUint32(r io.Reader) (x uint32, err error) { - if _, err = io.ReadFull(r, f.buf[:4]); err != nil { - return - } - x = binary.LittleEndian.Uint32(f.buf[:4]) - return -} diff --git a/vendor/github.com/pierrec/lz4/v4/options.go b/vendor/github.com/pierrec/lz4/v4/options.go index d9a3b4d07a8e2..4e1b6703b57d1 100644 --- a/vendor/github.com/pierrec/lz4/v4/options.go +++ b/vendor/github.com/pierrec/lz4/v4/options.go @@ -2,11 +2,10 @@ package lz4 import ( "fmt" - "reflect" - "runtime" - "github.com/pierrec/lz4/v4/internal/lz4block" "github.com/pierrec/lz4/v4/internal/lz4errors" + "reflect" + "runtime" ) //go:generate go run golang.org/x/tools/cmd/stringer -type=BlockSize,CompressionLevel -output options_gen.go @@ -82,7 +81,7 @@ func ChecksumOption(flag bool) Option { return func(a applier) error { switch w := a.(type) { case nil: - s := fmt.Sprintf("BlockChecksumOption(%v)", flag) + s := fmt.Sprintf("ChecksumOption(%v)", flag) return lz4errors.Error(s) case *Writer: w.frame.Descriptor.Flags.ContentChecksumSet(flag) @@ -112,16 +111,19 @@ func SizeOption(size uint64) Option { // ConcurrencyOption sets the number of go routines used for compression. // If n <= 0, then the output of runtime.GOMAXPROCS(0) is used. func ConcurrencyOption(n int) Option { + if n <= 0 { + n = runtime.GOMAXPROCS(0) + } return func(a applier) error { - switch w := a.(type) { + switch rw := a.(type) { case nil: s := fmt.Sprintf("ConcurrencyOption(%d)", n) return lz4errors.Error(s) case *Writer: - if n <= 0 { - n = runtime.GOMAXPROCS(0) - } - w.num = n + rw.num = n + return nil + case *Reader: + rw.num = n return nil } return lz4errors.ErrOptionNotApplicable @@ -179,9 +181,33 @@ func OnBlockDoneOption(handler func(size int)) Option { return lz4errors.Error(s) case *Writer: rw.handler = handler + return nil case *Reader: rw.handler = handler + return nil } - return nil + return lz4errors.ErrOptionNotApplicable + } +} + +// LegacyOption provides support for writing LZ4 frames in the legacy format. +// +// See https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md#legacy-frame. +// +// NB. compressed Linux kernel images use a tweaked LZ4 legacy format where +// the compressed stream is followed by the original (uncompressed) size of +// the kernel (https://events.static.linuxfound.org/sites/events/files/lcjpcojp13_klee.pdf). +// This is also supported as a special case. +func LegacyOption(legacy bool) Option { + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("LegacyOption(%v)", legacy) + return lz4errors.Error(s) + case *Writer: + rw.legacy = legacy + return nil + } + return lz4errors.ErrOptionNotApplicable } } diff --git a/vendor/github.com/pierrec/lz4/v4/reader.go b/vendor/github.com/pierrec/lz4/v4/reader.go index fb90a7fd8a453..403aaf697a3b7 100644 --- a/vendor/github.com/pierrec/lz4/v4/reader.go +++ b/vendor/github.com/pierrec/lz4/v4/reader.go @@ -3,6 +3,7 @@ package lz4 import ( "io" + "github.com/pierrec/lz4/v4/internal/lz4block" "github.com/pierrec/lz4/v4/internal/lz4errors" "github.com/pierrec/lz4/v4/internal/lz4stream" ) @@ -17,9 +18,13 @@ var readerStates = []aState{ // NewReader returns a new LZ4 frame decoder. func NewReader(r io.Reader) *Reader { + return newReader(r, false) +} + +func newReader(r io.Reader, legacy bool) *Reader { zr := &Reader{frame: lz4stream.NewFrame()} zr.state.init(readerStates) - _ = zr.Apply(defaultOnBlockDone) + _ = zr.Apply(DefaultConcurrency, defaultOnBlockDone) zr.Reset(r) return zr } @@ -28,10 +33,13 @@ func NewReader(r io.Reader) *Reader { type Reader struct { state _State src io.Reader // source reader + num int // concurrency level frame *lz4stream.Frame // frame being read - data []byte // pending data + data []byte // block buffer allocated in non concurrent mode + reads chan []byte // pending data idx int // size of pending data handler func(int) + cum uint32 } func (*Reader) private() {} @@ -64,8 +72,21 @@ func (r *Reader) Size() int { return 0 } +func (r *Reader) isNotConcurrent() bool { + return r.num == 1 +} + func (r *Reader) init() error { - return r.frame.InitR(r.src) + data, err := r.frame.InitR(r.src, r.num) + if err != nil { + return err + } + r.reads = data + r.idx = 0 + size := r.frame.Descriptor.Flags.BlockSizeIndex() + r.data = size.Get() + r.cum = 0 + return nil } func (r *Reader) Read(buf []byte) (n int, err error) { @@ -79,64 +100,78 @@ func (r *Reader) Read(buf []byte) (n int, err error) { if err = r.init(); r.state.next(err) { return } - size := r.frame.Descriptor.Flags.BlockSizeIndex() - r.data = size.Get() default: return 0, r.state.fail() } - if len(buf) == 0 { - return - } - - var bn int - if r.idx > 0 { - // Some left over data, use it. - goto fillbuf - } - // No uncompressed data yet. - r.data = r.data[:cap(r.data)] - for len(buf) >= len(r.data) { - // Input buffer large enough and no pending data: uncompress directly into it. - switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, buf); err { - case nil: - r.handler(bn) - n += bn - buf = buf[bn:] - case io.EOF: - goto close - default: - return + for len(buf) > 0 { + var bn int + if r.idx == 0 { + if r.isNotConcurrent() { + bn, err = r.read(buf) + } else { + lz4block.Put(r.data) + r.data = <-r.reads + if len(r.data) == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { + case nil: + case io.EOF: + if er := r.frame.CloseR(r.src); er != nil { + err = er + } + lz4block.Put(r.data) + r.data = nil + return + default: + return + } + } + if bn == 0 { + // Fill buf with buffered data. + bn = copy(buf, r.data[r.idx:]) + r.idx += bn + if r.idx == len(r.data) { + // All data read, get ready for the next Read. + r.idx = 0 + } } + buf = buf[bn:] + n += bn + r.handler(bn) } - if n > 0 { - // Some data was read, done for now. - return + return +} + +// read uncompresses the next block as follow: +// - if buf has enough room, the block is uncompressed into it directly +// and the lenght of used space is returned +// - else, the uncompress data is stored in r.data and 0 is returned +func (r *Reader) read(buf []byte) (int, error) { + block := r.frame.Blocks.Block + _, err := block.Read(r.frame, r.src, r.cum) + if err != nil { + return 0, err } - // Read the next block. - switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, r.data); err { - case nil: - r.handler(bn) - r.data = r.data[:bn] - goto fillbuf - case io.EOF: - default: - return + var direct bool + dst := r.data[:cap(r.data)] + if len(buf) >= len(dst) { + // Uncompress directly into buf. + direct = true + dst = buf } -close: - if er := r.frame.CloseR(r.src); er != nil { - err = er + dst, err = block.Uncompress(r.frame, dst, true) + if err != nil { + return 0, err } - r.Reset(nil) - return -fillbuf: - bn = copy(buf, r.data[r.idx:]) - n += bn - r.idx += bn - if r.idx == len(r.data) { - // All data read, get ready for the next Read. - r.idx = 0 + r.cum += uint32(len(dst)) + if direct { + return len(dst), nil } - return + r.data = dst + return 0, nil } // Reset clears the state of the Reader r such that it is equivalent to its @@ -145,13 +180,14 @@ fillbuf: // // w.Close must be called before Reset. func (r *Reader) Reset(reader io.Reader) { - size := r.frame.Descriptor.Flags.BlockSizeIndex() - size.Put(r.data) - r.frame.Reset(1) - r.src = reader - r.data = nil - r.idx = 0 + if r.data != nil { + lz4block.Put(r.data) + r.data = nil + } + r.frame.Reset(r.num) r.state.reset() + r.src = reader + r.reads = nil } // WriteTo efficiently uncompresses the data from the Reader underlying source to w. @@ -168,13 +204,28 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { } defer r.state.nextd(&err) - var bn int - block := r.frame.Blocks.Block - size := r.frame.Descriptor.Flags.BlockSizeIndex() - data := size.Get() - defer size.Put(data) + var data []byte + if r.isNotConcurrent() { + size := r.frame.Descriptor.Flags.BlockSizeIndex() + data = size.Get() + defer lz4block.Put(data) + } for { - switch bn, err = block.Uncompress(r.frame, r.src, data); err { + var bn int + var dst []byte + if r.isNotConcurrent() { + bn, err = r.read(data) + dst = data[:bn] + } else { + lz4block.Put(dst) + dst = <-r.reads + bn = len(dst) + if bn == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { case nil: case io.EOF: err = r.frame.CloseR(r.src) @@ -183,7 +234,7 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { return } r.handler(bn) - bn, err = w.Write(data[:bn]) + bn, err = w.Write(dst) n += int64(bn) if err != nil { return diff --git a/vendor/github.com/pierrec/lz4/v4/writer.go b/vendor/github.com/pierrec/lz4/v4/writer.go index e641be7c6a242..44a43d251b0d8 100644 --- a/vendor/github.com/pierrec/lz4/v4/writer.go +++ b/vendor/github.com/pierrec/lz4/v4/writer.go @@ -35,6 +35,7 @@ type Writer struct { data []byte // pending data idx int // size of pending data handler func(int) + legacy bool } func (*Writer) private() {} @@ -63,9 +64,11 @@ func (w *Writer) isNotConcurrent() bool { // init sets up the Writer when in newState. It does not change the Writer state. func (w *Writer) init() error { - w.frame.InitW(w.src, w.num) - size := w.frame.Descriptor.Flags.BlockSizeIndex() - w.data = size.Get() + w.frame.InitW(w.src, w.num, w.legacy) + if true || !w.isNotConcurrent() { + size := w.frame.Descriptor.Flags.BlockSizeIndex() + w.data = size.Get() + } w.idx = 0 return w.frame.Descriptor.Write(w.frame, w.src) } @@ -126,20 +129,19 @@ func (w *Writer) write(data []byte, safe bool) error { w.handler(len(block.Data)) return err } - size := w.frame.Descriptor.Flags.BlockSizeIndex() c := make(chan *lz4stream.FrameDataBlock) w.frame.Blocks.Blocks <- c - go func(c chan *lz4stream.FrameDataBlock, data []byte, size lz4block.BlockSizeIndex, safe bool) { - b := lz4stream.NewFrameDataBlock(size) + go func(c chan *lz4stream.FrameDataBlock, data []byte, safe bool) { + b := lz4stream.NewFrameDataBlock(w.frame) c <- b.Compress(w.frame, data, w.level) <-c w.handler(len(b.Data)) - b.CloseW(w.frame) + b.Close(w.frame) if safe { // safe to put it back as the last usage of it was FrameDataBlock.Write() called before c is closed - size.Put(data) + lz4block.Put(data) } - }(c, data, size, safe) + }(c, data, safe) return nil } @@ -165,8 +167,7 @@ func (w *Writer) Close() (err error) { err = w.frame.CloseW(w.src, w.num) // It is now safe to free the buffer. if w.data != nil { - size := w.frame.Descriptor.Flags.BlockSizeIndex() - size.Put(w.data) + lz4block.Put(w.data) w.data = nil } return @@ -204,13 +205,13 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { data := size.Get() if w.isNotConcurrent() { // Keep the same buffer for the whole process. - defer size.Put(data) + defer lz4block.Put(data) } for !done { rn, err = io.ReadFull(r, data) switch err { case nil: - case io.EOF: + case io.EOF, io.ErrUnexpectedEOF: // read may be partial done = true default: return diff --git a/vendor/github.com/ulikunitz/xz/SECURITY.md b/vendor/github.com/ulikunitz/xz/SECURITY.md new file mode 100644 index 0000000000000..5f7ec01b3b616 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/SECURITY.md @@ -0,0 +1,10 @@ +# Security Policy + +## Supported Versions + +Currently the last minor version v0.5.x is supported. + +## Reporting a Vulnerability + +Report a vulnerability by creating a Github issue at +. Expect a response in a week. diff --git a/vendor/github.com/ulikunitz/xz/TODO.md b/vendor/github.com/ulikunitz/xz/TODO.md index a4224ce142d88..88c7341c828d8 100644 --- a/vendor/github.com/ulikunitz/xz/TODO.md +++ b/vendor/github.com/ulikunitz/xz/TODO.md @@ -8,19 +8,17 @@ 1. Review encoder and check for lzma improvements under xz. 2. Fix binary tree matcher. -3. Compare compression ratio with xz tool using comparable parameters - and optimize parameters -4. Do some optimizations - - rename operation action and make it a simple type of size 8 - - make maxMatches, wordSize parameters - - stop searching after a certain length is found (parameter sweetLen) +3. Compare compression ratio with xz tool using comparable parameters and optimize parameters +4. rename operation action and make it a simple type of size 8 +5. make maxMatches, wordSize parameters +6. stop searching after a certain length is found (parameter sweetLen) ## Release v0.7 1. Optimize code 2. Do statistical analysis to get linear presets. 3. Test sync.Pool compatability for xz and lzma Writer and Reader -3. Fuzz optimized code. +4. Fuzz optimized code. ## Release v0.8 @@ -44,52 +42,77 @@ ## Package lzma -### Release v0.6 - -- Rewrite Encoder into a simple greedy one-op-at-a-time encoder - including - + simple scan at the dictionary head for the same byte - + use the killer byte (requiring matches to get longer, the first - test should be the byte that would make the match longer) +### v0.6 +* Rewrite Encoder into a simple greedy one-op-at-a-time encoder including + * simple scan at the dictionary head for the same byte + * use the killer byte (requiring matches to get longer, the first test should be the byte that would make the match longer) ## Optimizations -- There may be a lot of false sharing in lzma.State; check whether this - can be improved by reorganizing the internal structure of it. -- Check whether batching encoding and decoding improves speed. +* There may be a lot of false sharing in lzma. State; check whether this can be improved by reorganizing the internal structure of it. + +* Check whether batching encoding and decoding improves speed. ### DAG optimizations -- Use full buffer to create minimal bit-length above range encoder. -- Might be too slow (see v0.4) +* Use full buffer to create minimal bit-length above range encoder. +* Might be too slow (see v0.4) ### Different match finders -- hashes with 2, 3 characters additional to 4 characters -- binary trees with 2-7 characters (uint64 as key, use uint32 as +* hashes with 2, 3 characters additional to 4 characters +* binary trees with 2-7 characters (uint64 as key, use uint32 as + pointers into a an array) -- rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers + +* rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers + into an array with bit-steeling for the colors) ## Release Procedure -- execute goch -l for all packages; probably with lower param like 0.5. -- check orthography with gospell -- Write release notes in doc/relnotes. -- Update README.md -- xb copyright . in xz directory to ensure all new files have Copyright - header -- VERSION= go generate github.com/ulikunitz/xz/... to update - version files -- Execute test for Linux/amd64, Linux/x86 and Windows/amd64. -- Update TODO.md - write short log entry -- git checkout master && git merge dev -- git tag -a -- git push +* execute goch -l for all packages; probably with lower param like 0.5. +* check orthography with gospell +* Write release notes in doc/relnotes. +* Update README.md +* xb copyright . in xz directory to ensure all new files have Copyright header +* `VERSION= go generate github.com/ulikunitz/xz/...` to update version files +* Execute test for Linux/amd64, Linux/x86 and Windows/amd64. +* Update TODO.md - write short log entry +* `git checkout master && git merge dev` +* `git tag -a ` +* `git push` ## Log +### 2020-12-17 + +Release v0.5.9 fixes warnings, a typo and adds SECURITY.md. + +One fix is interesting. + +```go +const ( + a byte = 0x1 + b = 0x2 +) +``` + +The constants a and b don't have the same type. Correct is + +```go +const ( + a byte = 0x1 + b byte = 0x2 +) +``` + +### 2020-08-19 + +Release v0.5.8 fixes issue +[issue #35](https://github.com/ulikunitz/xz/issues/35). + ### 2020-02-24 Release v0.5.7 supports the check-ID None and fixes @@ -203,8 +226,8 @@ MININT. ### 2015-06-04 -It has been a productive day. I improved the interface of lzma.Reader -and lzma.Writer and fixed the error handling. +It has been a productive day. I improved the interface of lzma. Reader +and lzma. Writer and fixed the error handling. ### 2015-06-01 @@ -255,7 +278,7 @@ needed anymore. However I will implement a ReaderState and WriterState type to use static typing to ensure the right State object is combined with the -right lzbase.Reader and lzbase.Writer. +right lzbase. Reader and lzbase. Writer. As a start I have implemented ReaderState and WriterState to ensure that the state for reading is only used by readers and WriterState only @@ -277,11 +300,11 @@ old lzma package has been completely removed. ### 2015-04-05 -Implemented lzma.Reader and tested it. +Implemented lzma. Reader and tested it. ### 2015-04-04 -Implemented baseReader by adapting code form lzma.Reader. +Implemented baseReader by adapting code form lzma. Reader. ### 2015-04-03 @@ -297,7 +320,7 @@ However in Francesco Campoy's presentation "Go for Javaneros (Javaïstes?)" is the the idea that using an embedded field E, all the methods of E will be defined on T. If E is an interface T satisfies E. -https://talks.golang.org/2014/go4java.slide#51 + I have never used this, but it seems to be a cool idea. @@ -322,11 +345,11 @@ and the opCodec. 1. Implemented simple lzmago tool 2. Tested tool against large 4.4G file - - compression worked correctly; tested decompression with lzma - - decompression hits a full buffer condition + * compression worked correctly; tested decompression with lzma + * decompression hits a full buffer condition 3. Fixed a bug in the compressor and wrote a test for it 4. Executed full cycle for 4.4 GB file; performance can be improved ;-) ### 2015-01-11 -- Release v0.2 because of the working LZMA encoder and decoder +* Release v0.2 because of the working LZMA encoder and decoder diff --git a/vendor/github.com/ulikunitz/xz/bits.go b/vendor/github.com/ulikunitz/xz/bits.go index 364213dd94b33..dc8f32860044f 100644 --- a/vendor/github.com/ulikunitz/xz/bits.go +++ b/vendor/github.com/ulikunitz/xz/bits.go @@ -54,6 +54,8 @@ var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer") // readUvarint reads a uvarint from the given byte reader. func readUvarint(r io.ByteReader) (x uint64, n int, err error) { + const maxUvarintLen = 10 + var s uint i := 0 for { @@ -62,8 +64,11 @@ func readUvarint(r io.ByteReader) (x uint64, n int, err error) { return x, i, err } i++ + if i > maxUvarintLen { + return x, i, errOverflowU64 + } if b < 0x80 { - if i > 10 || i == 10 && b > 1 { + if i == maxUvarintLen && b > 1 { return x, i, errOverflowU64 } return x | uint64(b)<>27]) } +*/ // nlz32 computes the number of leading zeros for an unsigned 32-bit integer. func nlz32(x uint32) int { diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoder.go b/vendor/github.com/ulikunitz/xz/lzma/decoder.go index e5a760a50b0b8..4b820792a04e6 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/decoder.go +++ b/vendor/github.com/ulikunitz/xz/lzma/decoder.go @@ -200,7 +200,7 @@ func (d *decoder) decompress() error { op, err := d.readOp() switch err { case nil: - break + // break case errEOS: d.eos = true if !d.rd.possiblyAtEnd() { diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go index ba06712b03f1c..dd44e6625cc6f 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go +++ b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go @@ -126,10 +126,3 @@ func (d *decoderDict) Available() int { return d.buf.Available() } // Read reads data from the buffer contained in the decoder dictionary. func (d *decoderDict) Read(p []byte) (n int, err error) { return d.buf.Read(p) } - -// Buffered returns the number of bytes currently buffered in the -// decoder dictionary. -func (d *decoderDict) buffered() int { return d.buf.Buffered() } - -// Peek gets data from the buffer without advancing the rear index. -func (d *decoderDict) peek(p []byte) (n int, err error) { return d.buf.Peek(p) } diff --git a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go index e6e0c6ddf369e..064642831628c 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go +++ b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go @@ -4,21 +4,10 @@ package lzma -import "fmt" - // directCodec allows the encoding and decoding of values with a fixed number // of bits. The number of bits must be in the range [1,32]. type directCodec byte -// makeDirectCodec creates a directCodec. The function panics if the number of -// bits is not in the range [1,32]. -func makeDirectCodec(bits int) directCodec { - if !(1 <= bits && bits <= 32) { - panic(fmt.Errorf("bits=%d out of range", bits)) - } - return directCodec(bits) -} - // Bits returns the number of bits supported by this codec. func (dc directCodec) Bits() int { return int(dc) diff --git a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go index 69871c04ad5eb..9ed486d27f6a9 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go +++ b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go @@ -20,8 +20,6 @@ const ( posSlotBits = 6 // number of align bits alignBits = 4 - // maximum position slot - maxPosSlot = 63 ) // distCodec provides encoding and decoding of distance values. @@ -45,20 +43,6 @@ func (dc *distCodec) deepcopy(src *distCodec) { dc.alignCodec.deepcopy(&src.alignCodec) } -// distBits returns the number of bits required to encode dist. -func distBits(dist uint32) int { - if dist < startPosModel { - return 6 - } - // slot s > 3, dist d - // s = 2(bits(d)-1) + bit(d, bits(d)-2) - // s>>1 = bits(d)-1 - // bits(d) = 32-nlz32(d) - // s>>1=31-nlz32(d) - // n = 5 + (s>>1) = 36 - nlz32(d) - return 36 - nlz32(dist) -} - // newDistCodec creates a new distance codec. func (dc *distCodec) init() { for i := range dc.posSlotCodecs { diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go index 40f3d3f64a706..c36308d7c5635 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go +++ b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go @@ -19,7 +19,7 @@ type matcher interface { } // encoderDict provides the dictionary of the encoder. It includes an -// addtional buffer atop of the actual dictionary. +// additional buffer atop of the actual dictionary. type encoderDict struct { buf buffer m matcher diff --git a/vendor/github.com/ulikunitz/xz/lzma/header2.go b/vendor/github.com/ulikunitz/xz/lzma/header2.go index cd148812cd510..ffeca35c3ee38 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/header2.go +++ b/vendor/github.com/ulikunitz/xz/lzma/header2.go @@ -264,7 +264,7 @@ type chunkState byte // state const ( start chunkState = 'S' - stop = 'T' + stop chunkState = 'T' ) // errors for the chunk state handling diff --git a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go index 927395bd8097d..35b0640642fe4 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go +++ b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go @@ -56,19 +56,6 @@ func (lc *lengthCodec) init() { lc.high = makeTreeCodec(8) } -// lBits gives the number of bits used for the encoding of the l value -// provided to the range encoder. -func lBits(l uint32) int { - switch { - case l < 8: - return 4 - case l < 16: - return 5 - default: - return 10 - } -} - // Encode encodes the length offset. The length offset l can be compute by // subtracting minMatchLen (2) from the actual length. // diff --git a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go index ca31530fd574b..7b1ad1d9bf9c9 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go +++ b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go @@ -123,10 +123,3 @@ const ( minLP = 0 maxLP = 4 ) - -// minState and maxState define a range for the state values stored in -// the State values. -const ( - minState = 0 - maxState = 11 -) diff --git a/vendor/github.com/ulikunitz/xz/lzma/operation.go b/vendor/github.com/ulikunitz/xz/lzma/operation.go index a75c9b46c52d2..2f9b78ea51023 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/operation.go +++ b/vendor/github.com/ulikunitz/xz/lzma/operation.go @@ -5,7 +5,6 @@ package lzma import ( - "errors" "fmt" "unicode" ) @@ -24,30 +23,6 @@ type match struct { n int } -// verify checks whether the match is valid. If that is not the case an -// error is returned. -func (m match) verify() error { - if !(minDistance <= m.distance && m.distance <= maxDistance) { - return errors.New("distance out of range") - } - if !(1 <= m.n && m.n <= maxMatchLen) { - return errors.New("length out of range") - } - return nil -} - -// l return the l-value for the match, which is the difference of length -// n and 2. -func (m match) l() uint32 { - return uint32(m.n - minMatchLen) -} - -// dist returns the dist value for the match, which is one less of the -// distance stored in the match. -func (m match) dist() uint32 { - return uint32(m.distance - minDistance) -} - // Len returns the number of bytes matched. func (m match) Len() int { return m.n diff --git a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go index 7189a0377649d..7b299abfefd97 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go +++ b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go @@ -131,32 +131,6 @@ type rangeDecoder struct { code uint32 } -// init initializes the range decoder, by reading from the byte reader. -func (d *rangeDecoder) init() error { - d.nrange = 0xffffffff - d.code = 0 - - b, err := d.br.ReadByte() - if err != nil { - return err - } - if b != 0 { - return errors.New("newRangeDecoder: first byte not zero") - } - - for i := 0; i < 4; i++ { - if err = d.updateCode(); err != nil { - return err - } - } - - if d.code >= d.nrange { - return errors.New("newRangeDecoder: d.code >= d.nrange") - } - - return nil -} - // newRangeDecoder initializes a range decoder. It reads five bytes from the // reader and therefore may return an error. func newRangeDecoder(br io.ByteReader) (d *rangeDecoder, err error) { diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader2.go b/vendor/github.com/ulikunitz/xz/lzma/reader2.go index 33074e6242975..e34c23f9c0d0e 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/reader2.go +++ b/vendor/github.com/ulikunitz/xz/lzma/reader2.go @@ -48,7 +48,6 @@ type Reader2 struct { chunkReader io.Reader cstate chunkState - ctype chunkType } // NewReader2 creates a reader for an LZMA2 chunk sequence. diff --git a/vendor/github.com/ulikunitz/xz/lzma/state.go b/vendor/github.com/ulikunitz/xz/lzma/state.go index 03f061cf109e0..fbe3a39425590 100644 --- a/vendor/github.com/ulikunitz/xz/lzma/state.go +++ b/vendor/github.com/ulikunitz/xz/lzma/state.go @@ -53,12 +53,6 @@ func (s *state) Reset() { s.distCodec.init() } -// initState initializes the state. -func initState(s *state, p Properties) { - *s = state{Properties: p} - s.Reset() -} - // newState creates a new state from the give Properties. func newState(p Properties) *state { s := &state{Properties: p} diff --git a/vendor/github.com/ulikunitz/xz/reader.go b/vendor/github.com/ulikunitz/xz/reader.go index 22cd6d5007f01..795858914da8c 100644 --- a/vendor/github.com/ulikunitz/xz/reader.go +++ b/vendor/github.com/ulikunitz/xz/reader.go @@ -26,13 +26,6 @@ type ReaderConfig struct { SingleStream bool } -// fill replaces all zero values with their default values. -func (c *ReaderConfig) fill() { - if c.DictCap == 0 { - c.DictCap = 8 * 1024 * 1024 - } -} - // Verify checks the reader parameters for Validity. Zero values will be // replaced by default values. func (c *ReaderConfig) Verify() error { @@ -165,9 +158,6 @@ func (c ReaderConfig) newStreamReader(xz io.Reader) (r *streamReader, err error) return r, nil } -// errIndex indicates an error with the xz file index. -var errIndex = errors.New("xz: error in xz file index") - // readTail reads the index body and the xz footer. func (r *streamReader) readTail() error { index, n, err := readIndexBody(r.xz) @@ -265,7 +255,6 @@ type blockReader struct { n int64 hash hash.Hash r io.Reader - err error } // newBlockReader creates a new block reader. @@ -315,10 +304,6 @@ func (br *blockReader) record() record { return record{br.unpaddedSize(), br.uncompressedSize()} } -// errBlockSize indicates that the size of the block in the block header -// is wrong. -var errBlockSize = errors.New("xz: wrong uncompressed size for block") - // Read reads data from the block. func (br *blockReader) Read(p []byte) (n int, err error) { n, err = br.r.Read(p) diff --git a/vendor/github.com/ulikunitz/xz/writer.go b/vendor/github.com/ulikunitz/xz/writer.go index aec10dfa62477..a9ed44912503f 100644 --- a/vendor/github.com/ulikunitz/xz/writer.go +++ b/vendor/github.com/ulikunitz/xz/writer.go @@ -6,6 +6,7 @@ package xz import ( "errors" + "fmt" "hash" "io" @@ -190,6 +191,9 @@ func (c WriterConfig) NewWriter(xz io.Writer) (w *Writer, err error) { return nil, err } data, err := w.h.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("w.h.MarshalBinary(): error %w", err) + } if _, err = xz.Write(data); err != nil { return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 6b742e2f59c45..f32aaed460083 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -129,7 +129,7 @@ github.com/aliyun/alibaba-cloud-sdk-go/services/sts # github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5 ## explicit github.com/aliyun/aliyun-oss-go-sdk/oss -# github.com/andybalholm/brotli v1.0.0 +# github.com/andybalholm/brotli v1.0.1 github.com/andybalholm/brotli # github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230 github.com/apache/arrow/go/arrow @@ -350,7 +350,7 @@ github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig # github.com/docker/go-units v0.4.0 github.com/docker/go-units -# github.com/dsnet/compress v0.0.1 +# github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 github.com/dsnet/compress github.com/dsnet/compress/bzip2 github.com/dsnet/compress/bzip2/internal/sais @@ -871,15 +871,16 @@ github.com/keybase/go-crypto/openpgp/errors github.com/keybase/go-crypto/openpgp/packet github.com/keybase/go-crypto/openpgp/s2k github.com/keybase/go-crypto/rsa -# github.com/klauspost/compress v1.10.10 +# github.com/klauspost/compress v1.11.4 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/gzip github.com/klauspost/compress/huff0 github.com/klauspost/compress/snappy +github.com/klauspost/compress/zip github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/klauspost/pgzip v1.2.4 +# github.com/klauspost/pgzip v1.2.5 github.com/klauspost/pgzip # github.com/konsorten/go-windows-terminal-sequences v1.0.3 github.com/konsorten/go-windows-terminal-sequences @@ -907,7 +908,7 @@ github.com/mattn/go-isatty github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/mholt/archiver/v3 v3.5.0 +# github.com/mholt/archiver/v3 v3.5.1 ## explicit github.com/mholt/archiver/v3 # github.com/michaelklishin/rabbit-hole v0.0.0-20191008194146-93d9988f0cd5 @@ -1054,7 +1055,7 @@ github.com/petermattis/goid # github.com/pierrec/lz4 v2.5.2+incompatible github.com/pierrec/lz4 github.com/pierrec/lz4/internal/xxh32 -# github.com/pierrec/lz4/v4 v4.0.3 +# github.com/pierrec/lz4/v4 v4.1.2 github.com/pierrec/lz4/v4 github.com/pierrec/lz4/v4/internal/lz4block github.com/pierrec/lz4/v4/internal/lz4errors @@ -1167,7 +1168,7 @@ github.com/tklauser/go-sysconf github.com/tklauser/numcpus # github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 github.com/tv42/httpunix -# github.com/ulikunitz/xz v0.5.7 +# github.com/ulikunitz/xz v0.5.9 github.com/ulikunitz/xz github.com/ulikunitz/xz/internal/hash github.com/ulikunitz/xz/internal/xlog