Updates to Brotli compression format, decoder and encoder
This commit contains a batch of changes that were made to the Brotli compression algorithm in the last three weeks. Most important changes: * Added UTF8 context model for good text compression. * Simplified context modeling by having only 4 context modes. * Per-block context mode selection. * Faster backward copying and bit reading functions. * More efficient histogram coding. * Streaming support for the decoder and encoder.
This commit is contained in:
Zoltan Szabadka
+9
-83
@@ -15,6 +15,7 @@
|
||||
// Bit reading helpers
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "./bit_reader.h"
|
||||
|
||||
@@ -22,99 +23,24 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define MAX_NUM_BIT_READ 25
|
||||
|
||||
#define LBITS 64 // Number of bits prefetched.
|
||||
#define WBITS 32 // Minimum number of bytes needed after
|
||||
// BrotliFillBitWindow.
|
||||
#define LOG8_WBITS 4 // Number of bytes needed to store WBITS bits.
|
||||
|
||||
static const uint32_t kBitMask[MAX_NUM_BIT_READ] = {
|
||||
0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767,
|
||||
65535, 131071, 262143, 524287, 1048575, 2097151, 4194303, 8388607, 16777215
|
||||
};
|
||||
|
||||
void BrotliInitBitReader(BrotliBitReader* const br,
|
||||
const uint8_t* const start,
|
||||
size_t length) {
|
||||
int BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input) {
|
||||
size_t i;
|
||||
assert(br != NULL);
|
||||
assert(start != NULL);
|
||||
assert(length < 0xfffffff8u); // can't happen with a RIFF chunk.
|
||||
|
||||
br->buf_ = start;
|
||||
br->len_ = length;
|
||||
br->input_ = input;
|
||||
br->val_ = 0;
|
||||
br->pos_ = 0;
|
||||
br->bit_pos_ = 0;
|
||||
br->end_pos_ = 0;
|
||||
br->eos_ = 0;
|
||||
br->error_ = 0;
|
||||
for (i = 0; i < sizeof(br->val_) && i < br->len_; ++i) {
|
||||
if (!BrotliReadMoreInput(br)) {
|
||||
return 0;
|
||||
}
|
||||
for (i = 0; i < sizeof(br->val_); ++i) {
|
||||
br->val_ |= ((uint64_t)br->buf_[br->pos_]) << (8 * i);
|
||||
++br->pos_;
|
||||
}
|
||||
}
|
||||
|
||||
void BrotliBitReaderSetBuffer(BrotliBitReader* const br,
|
||||
const uint8_t* const buf, size_t len) {
|
||||
assert(br != NULL);
|
||||
assert(buf != NULL);
|
||||
assert(len < 0xfffffff8u); // can't happen with a RIFF chunk.
|
||||
br->eos_ = (br->pos_ >= len);
|
||||
br->buf_ = buf;
|
||||
br->len_ = len;
|
||||
}
|
||||
|
||||
// If not at EOS, reload up to LBITS byte-by-byte
|
||||
static void ShiftBytes(BrotliBitReader* const br) {
|
||||
while (br->bit_pos_ >= 8 && br->pos_ < br->len_) {
|
||||
br->val_ >>= 8;
|
||||
br->val_ |= ((uint64_t)br->buf_[br->pos_]) << (LBITS - 8);
|
||||
++br->pos_;
|
||||
br->bit_pos_ -= 8;
|
||||
}
|
||||
}
|
||||
|
||||
void BrotliFillBitWindow(BrotliBitReader* const br) {
|
||||
if (br->bit_pos_ >= WBITS) {
|
||||
#if (defined(__x86_64__) || defined(_M_X64))
|
||||
if (br->pos_ + sizeof(br->val_) < br->len_) {
|
||||
br->val_ >>= WBITS;
|
||||
br->bit_pos_ -= WBITS;
|
||||
// The expression below needs a little-endian arch to work correctly.
|
||||
// This gives a large speedup for decoding speed.
|
||||
br->val_ |= *(const uint64_t*)(br->buf_ + br->pos_) << (LBITS - WBITS);
|
||||
br->pos_ += LOG8_WBITS;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ShiftBytes(br); // Slow path.
|
||||
if (br->pos_ == br->len_ && br->bit_pos_ == LBITS) {
|
||||
br->eos_ = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t BrotliReadBits(BrotliBitReader* const br, int n_bits) {
|
||||
assert(n_bits >= 0);
|
||||
// Flag an error if end_of_stream or n_bits is more than allowed limit.
|
||||
if (n_bits == 0 || (!br->eos_ && n_bits < MAX_NUM_BIT_READ)) {
|
||||
const uint32_t val =
|
||||
(uint32_t)(br->val_ >> br->bit_pos_) & kBitMask[n_bits];
|
||||
const int new_bits = br->bit_pos_ + n_bits;
|
||||
br->bit_pos_ = new_bits;
|
||||
// If this read is going to cross the read buffer, set the eos flag.
|
||||
if (br->pos_ == br->len_) {
|
||||
if (new_bits >= LBITS) {
|
||||
br->eos_ = 1;
|
||||
}
|
||||
}
|
||||
ShiftBytes(br);
|
||||
return val;
|
||||
} else {
|
||||
br->error_ = 1;
|
||||
return 0;
|
||||
}
|
||||
return (br->end_pos_ > 0);
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
|
||||
+110
-21
@@ -17,34 +17,39 @@
|
||||
#ifndef BROTLI_DEC_BIT_READER_H_
|
||||
#define BROTLI_DEC_BIT_READER_H_
|
||||
|
||||
#include <string.h>
|
||||
#include "./streams.h"
|
||||
#include "./types.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define BROTLI_MAX_NUM_BIT_READ 25
|
||||
#define BROTLI_READ_SIZE 4096
|
||||
#define BROTLI_IBUF_SIZE (2 * BROTLI_READ_SIZE + 32)
|
||||
#define BROTLI_IBUF_MASK (2 * BROTLI_READ_SIZE - 1)
|
||||
|
||||
#define UNALIGNED_COPY64(dst, src) *(uint64_t*)(dst) = *(const uint64_t*)(src)
|
||||
|
||||
static const uint32_t kBitMask[BROTLI_MAX_NUM_BIT_READ] = {
|
||||
0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767,
|
||||
65535, 131071, 262143, 524287, 1048575, 2097151, 4194303, 8388607, 16777215
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint64_t val_; // pre-fetched bits
|
||||
const uint8_t* buf_; // input byte buffer
|
||||
size_t len_; // buffer length
|
||||
size_t pos_; // byte position in buf_
|
||||
int bit_pos_; // current bit-reading position in val_
|
||||
int eos_; // bitstream is finished
|
||||
int error_; // an error occurred (buffer overflow attempt...)
|
||||
// Input byte buffer, consist of a ringbuffer and a "slack" region where
|
||||
// bytes from the start of the ringbuffer are copied.
|
||||
uint8_t buf_[BROTLI_IBUF_SIZE];
|
||||
BrotliInput input_; // input callback
|
||||
uint64_t val_; // pre-fetched bits
|
||||
size_t pos_; // byte position in stream
|
||||
int bit_pos_; // current bit-reading position in val_
|
||||
size_t end_pos_; // current end position in stream
|
||||
int eos_; // input stream is finished
|
||||
} BrotliBitReader;
|
||||
|
||||
void BrotliInitBitReader(BrotliBitReader* const br,
|
||||
const uint8_t* const start,
|
||||
size_t length);
|
||||
|
||||
// Sets a new data buffer.
|
||||
void BrotliBitReaderSetBuffer(BrotliBitReader* const br,
|
||||
const uint8_t* const buffer, size_t length);
|
||||
|
||||
// Reads the specified number of bits from Read Buffer.
|
||||
// Flags an error in case end_of_stream or n_bits is more than allowed limit.
|
||||
// Flags eos if this read attempt is going to cross the read buffer.
|
||||
uint32_t BrotliReadBits(BrotliBitReader* const br, int n_bits);
|
||||
int BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input);
|
||||
|
||||
// Return the prefetched bits, so they can be looked up.
|
||||
static BROTLI_INLINE uint32_t BrotliPrefetchBits(BrotliBitReader* const br) {
|
||||
@@ -57,8 +62,92 @@ static BROTLI_INLINE void BrotliSetBitPos(BrotliBitReader* const br, int val) {
|
||||
br->bit_pos_ = val;
|
||||
}
|
||||
|
||||
// Advances the Read buffer by 4 bytes to make room for reading next 32 bits.
|
||||
void BrotliFillBitWindow(BrotliBitReader* const br);
|
||||
// Reload up to 64 bits byte-by-byte
|
||||
static BROTLI_INLINE void ShiftBytes(BrotliBitReader* const br) {
|
||||
while (br->bit_pos_ >= 8) {
|
||||
br->val_ >>= 8;
|
||||
br->val_ |= ((uint64_t)br->buf_[br->pos_ & BROTLI_IBUF_MASK]) << 56;
|
||||
++br->pos_;
|
||||
br->bit_pos_ -= 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Fills up the input ringbuffer by calling the input callback.
|
||||
//
|
||||
// Does nothing if there are at least 32 bytes present after current position.
|
||||
//
|
||||
// Returns 0 if either:
|
||||
// - the input callback returned an error, or
|
||||
// - there is no more input and the position is past the end of the stream.
|
||||
//
|
||||
// After encountering the end of the input stream, 32 additional zero bytes are
|
||||
// copied to the ringbuffer, therefore it is safe to call this function after
|
||||
// every 32 bytes of input is read.
|
||||
static BROTLI_INLINE int BrotliReadMoreInput(BrotliBitReader* const br) {
|
||||
if (br->pos_ + 32 < br->end_pos_) {
|
||||
return 1;
|
||||
} else if (br->eos_) {
|
||||
return (br->pos_ << 3) + br->bit_pos_ <= (br->end_pos_ << 3) + 64;
|
||||
} else {
|
||||
uint8_t* dst = br->buf_ + (br->end_pos_ & BROTLI_IBUF_MASK);
|
||||
int bytes_read = BrotliRead(br->input_, dst, BROTLI_READ_SIZE);
|
||||
if (bytes_read < 0) {
|
||||
return 0;
|
||||
}
|
||||
if (bytes_read < BROTLI_READ_SIZE) {
|
||||
br->eos_ = 1;
|
||||
// Store 32 bytes of zero after the stream end.
|
||||
#if (defined(__x86_64__) || defined(_M_X64))
|
||||
*(uint64_t*)(dst + bytes_read) = 0;
|
||||
*(uint64_t*)(dst + bytes_read + 8) = 0;
|
||||
*(uint64_t*)(dst + bytes_read + 16) = 0;
|
||||
*(uint64_t*)(dst + bytes_read + 24) = 0;
|
||||
#else
|
||||
memset(dst + bytes_read, 0, 32);
|
||||
#endif
|
||||
}
|
||||
if (dst == br->buf_) {
|
||||
// Copy the head of the ringbuffer to the slack region.
|
||||
#if (defined(__x86_64__) || defined(_M_X64))
|
||||
UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 32, br->buf_);
|
||||
UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 24, br->buf_ + 8);
|
||||
UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 16, br->buf_ + 16);
|
||||
UNALIGNED_COPY64(br->buf_ + BROTLI_IBUF_SIZE - 8, br->buf_ + 24);
|
||||
#else
|
||||
memcpy(br->buf_ + (BROTLI_READ_SIZE << 1), br->buf_, 32);
|
||||
#endif
|
||||
}
|
||||
br->end_pos_ += bytes_read;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Advances the Read buffer by 5 bytes to make room for reading next 24 bits.
|
||||
static BROTLI_INLINE void BrotliFillBitWindow(BrotliBitReader* const br) {
|
||||
if (br->bit_pos_ >= 40) {
|
||||
#if (defined(__x86_64__) || defined(_M_X64))
|
||||
br->val_ >>= 40;
|
||||
br->bit_pos_ -= 40;
|
||||
// The expression below needs a little-endian arch to work correctly.
|
||||
// This gives a large speedup for decoding speed.
|
||||
br->val_ |= *(const uint64_t*)(
|
||||
br->buf_ + (br->pos_ & BROTLI_IBUF_MASK)) << 24;
|
||||
br->pos_ += 5;
|
||||
#else
|
||||
ShiftBytes(br);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
// Reads the specified number of bits from Read Buffer.
|
||||
// Requires that n_bits is positive.
|
||||
static BROTLI_INLINE uint32_t BrotliReadBits(
|
||||
BrotliBitReader* const br, int n_bits) {
|
||||
BrotliFillBitWindow(br);
|
||||
const uint32_t val = (uint32_t)(br->val_ >> br->bit_pos_) & kBitMask[n_bits];
|
||||
br->bit_pos_ += n_bits;
|
||||
return val;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
|
||||
+217
-81
@@ -12,34 +12,154 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Lookup tables to map the previous one to three bytes to a context id.
|
||||
// Lookup table to map the previous two bytes to a context id.
|
||||
//
|
||||
// There are four different context modeling modes defined here:
|
||||
// CONTEXT_LSB6: context id is the least significant 6 bits of the last byte,
|
||||
// CONTEXT_MSB6: context id is the most significant 6 bits of the last byte,
|
||||
// CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text,
|
||||
// CONTEXT_SIGNED: second-order context model tuned for signed integers.
|
||||
//
|
||||
// The context id for the UTF8 context model is calculated as follows. If p1
|
||||
// and p2 are the previous two bytes, we calcualte the context as
|
||||
//
|
||||
// context = kContextLookup[p1] | kContextLookup[p2 + 256].
|
||||
//
|
||||
// If the previous two bytes are ASCII characters (i.e. < 128), this will be
|
||||
// equivalent to
|
||||
//
|
||||
// context = 4 * context1(p1) + context2(p2),
|
||||
//
|
||||
// where context1 is based on the previous byte in the following way:
|
||||
//
|
||||
// 0 : non-ASCII control
|
||||
// 1 : \t, \n, \r
|
||||
// 2 : space
|
||||
// 3 : other punctuation
|
||||
// 4 : " '
|
||||
// 5 : %
|
||||
// 6 : ( < [ {
|
||||
// 7 : ) > ] }
|
||||
// 8 : , ; :
|
||||
// 9 : .
|
||||
// 10 : =
|
||||
// 11 : number
|
||||
// 12 : upper-case vowel
|
||||
// 13 : upper-case consonant
|
||||
// 14 : lower-case vowel
|
||||
// 15 : lower-case consonant
|
||||
//
|
||||
// and context2 is based on the second last byte:
|
||||
//
|
||||
// 0 : control, space
|
||||
// 1 : punctuation
|
||||
// 2 : upper-case letter, number
|
||||
// 3 : lower-case letter
|
||||
//
|
||||
// If the last byte is ASCII, and the second last byte is not (in a valid UTF8
|
||||
// stream it will be a continuation byte, value between 128 and 191), the
|
||||
// context is the same as if the second last byte was an ASCII control or space.
|
||||
//
|
||||
// If the last byte is a UTF8 lead byte (value >= 192), then the next byte will
|
||||
// be a continuation byte and the context id is 2 or 3 depending on the LSB of
|
||||
// the last byte and to a lesser extent on the second last byte if it is ASCII.
|
||||
//
|
||||
// If the last byte is a UTF8 continuation byte, the second last byte can be:
|
||||
// - continuation byte: the next byte is probably ASCII or lead byte (assuming
|
||||
// 4-byte UTF8 characters are rare) and the context id is 0 or 1.
|
||||
// - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1
|
||||
// - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3
|
||||
//
|
||||
// The possible value combinations of the previous two bytes, the range of
|
||||
// context ids and the type of the next byte is summarized in the table below:
|
||||
//
|
||||
// |--------\-----------------------------------------------------------------|
|
||||
// | \ Last byte |
|
||||
// | Second \---------------------------------------------------------------|
|
||||
// | last byte \ ASCII | cont. byte | lead byte |
|
||||
// | \ (0-127) | (128-191) | (192-) |
|
||||
// |=============|===================|=====================|==================|
|
||||
// | ASCII | next: ASCII/lead | not valid | next: cont. |
|
||||
// | (0-127) | context: 4 - 63 | | context: 2 - 3 |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |
|
||||
// | (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | lead byte | not valid | next: ASCII/lead | not valid |
|
||||
// | (192-207) | | context: 0 - 1 | |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | lead byte | not valid | next: cont. | not valid |
|
||||
// | (208-) | | context: 2 - 3 | |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
//
|
||||
// The context id for the signed context mode is calculated as:
|
||||
//
|
||||
// context = (kContextLookup[512 + p1] << 3) | kContextLookup[512 + p2].
|
||||
//
|
||||
// For any context modeling modes, the context ids can be calculated by |-ing
|
||||
// together two lookups from one table using context model dependent offsets:
|
||||
//
|
||||
// context = kContextLookup[offset1 + p1] | kContextLookup[offset2 + p2].
|
||||
//
|
||||
// where offset1 and offset2 are dependent on the context mode.
|
||||
|
||||
#ifndef BROTLI_DEC_CONTEXT_H_
|
||||
#define BROTLI_DEC_CONTEXT_H_
|
||||
|
||||
|
||||
#include "./types.h"
|
||||
|
||||
static const int kSigned2BitContextLookup[] = {
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
|
||||
enum ContextType {
|
||||
CONTEXT_LSB6 = 0,
|
||||
CONTEXT_MSB6 = 1,
|
||||
CONTEXT_UTF8 = 2,
|
||||
CONTEXT_SIGNED = 3
|
||||
};
|
||||
|
||||
static const int kSigned3BitContextLookup[] = {
|
||||
// Common context lookup table for all context modes.
|
||||
static const uint8_t kContextLookup[1792] = {
|
||||
// CONTEXT_UTF8, last byte.
|
||||
//
|
||||
// ASCII range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
|
||||
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
|
||||
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
|
||||
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
|
||||
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
|
||||
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
|
||||
// UTF8 continuation byte range.
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
// UTF8 lead byte range.
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
// CONTEXT_UTF8 second last byte.
|
||||
//
|
||||
// ASCII range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
||||
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
|
||||
// UTF8 continuation byte range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
// UTF8 lead byte range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
// CONTEXT_SIGNED, second last byte.
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
@@ -56,69 +176,85 @@ static const int kSigned3BitContextLookup[] = {
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
|
||||
// CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits.
|
||||
0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56,
|
||||
// CONTEXT_LSB6, last byte.
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
|
||||
// CONTEXT_MSB6, last byte.
|
||||
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
|
||||
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15,
|
||||
16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27,
|
||||
28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31,
|
||||
32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35,
|
||||
36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43,
|
||||
44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47,
|
||||
48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51,
|
||||
52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55,
|
||||
56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59,
|
||||
60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63,
|
||||
// CONTEXT_{M,L}SB6, second last byte,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
};
|
||||
|
||||
static const int kSigned4BitContextLookup[] = {
|
||||
0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15,
|
||||
static const int kContextLookupOffsets[8] = {
|
||||
// CONTEXT_LSB6
|
||||
1024, 1536,
|
||||
// CONTEXT_MSB6
|
||||
1280, 1536,
|
||||
// CONTEXT_UTF8
|
||||
0, 256,
|
||||
// CONTEXT_SIGNED
|
||||
768, 512,
|
||||
};
|
||||
|
||||
enum ContextType {
|
||||
CONTEXT_FULL = 0,
|
||||
CONTEXT_MSB7 = 1,
|
||||
CONTEXT_MSB6 = 2,
|
||||
CONTEXT_MSB5 = 3,
|
||||
CONTEXT_MSB4 = 4,
|
||||
CONTEXT_MSB3 = 5,
|
||||
CONTEXT_MSB2 = 6,
|
||||
CONTEXT_MSB1 = 7,
|
||||
CONTEXT_IS_ZERO = 8,
|
||||
CONTEXT_SIGNED_2BIT = 9,
|
||||
CONTEXT_SIGNED_3BIT = 10,
|
||||
CONTEXT_SIGNED_4BIT = 11,
|
||||
CONTEXT_SIGNED_MIXED_3BYTE = 12
|
||||
};
|
||||
|
||||
static const int kContextSize[] = {
|
||||
256, 128, 64, 32, 16, 8, 4, 2, 2, 4, 8, 16, 64,
|
||||
};
|
||||
|
||||
static BROTLI_INLINE int NumContexts(int mode) {
|
||||
return kContextSize[mode];
|
||||
}
|
||||
|
||||
static BROTLI_INLINE uint8_t Context(uint8_t prev_byte, uint8_t prev_byte2,
|
||||
uint8_t prev_byte3, int mode) {
|
||||
switch (mode) {
|
||||
case CONTEXT_IS_ZERO:
|
||||
return prev_byte == 0 ? 0 : 1;
|
||||
case CONTEXT_SIGNED_2BIT:
|
||||
return kSigned2BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_3BIT:
|
||||
return kSigned3BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_4BIT:
|
||||
return kSigned4BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_MIXED_3BYTE:
|
||||
return ((kSigned3BitContextLookup[prev_byte] << 3) +
|
||||
(kSigned2BitContextLookup[prev_byte2] << 1) +
|
||||
(prev_byte3 == 0 ? 0 : 1));
|
||||
default:
|
||||
return prev_byte >> mode;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // BROTLI_DEC_CONTEXT_H_
|
||||
|
||||
+436
-285
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,7 @@
|
||||
#ifndef BROTLI_DEC_DECODE_H_
|
||||
#define BROTLI_DEC_DECODE_H_
|
||||
|
||||
#include "./streams.h"
|
||||
#include "./types.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
@@ -39,6 +40,10 @@ int BrotliDecompressBuffer(size_t encoded_size,
|
||||
size_t* decoded_size,
|
||||
uint8_t* decoded_buffer);
|
||||
|
||||
// Same as above, but uses the specified input and output callbacks instead of
|
||||
// reading from and writing to pre-allocated memory buffers.
|
||||
int BrotliDecompress(BrotliInput input, BrotliOutput output);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
+16
-69
@@ -24,10 +24,6 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// Uncomment the following to use look-up table for ReverseBits()
|
||||
// (might be faster on some platform)
|
||||
// #define USE_LUT_REVERSE_BITS
|
||||
|
||||
#define NON_EXISTENT_SYMBOL (-1)
|
||||
#define MAX_ALLOWED_CODE_LENGTH 15
|
||||
|
||||
@@ -55,7 +51,6 @@ static void AssignChildren(HuffmanTree* const tree,
|
||||
|
||||
static int TreeInit(HuffmanTree* const tree, int num_leaves) {
|
||||
assert(tree != NULL);
|
||||
tree->fixed_bit_length_ = 0;
|
||||
if (num_leaves == 0) return 0;
|
||||
// We allocate maximum possible nodes in the tree at once.
|
||||
// Note that a Huffman tree is a full binary tree; and in a full binary tree
|
||||
@@ -84,7 +79,7 @@ void BrotliHuffmanTreeRelease(HuffmanTree* const tree) {
|
||||
// Utility: converts Huffman code lengths to corresponding Huffman codes.
|
||||
// 'huff_codes' should be pre-allocated.
|
||||
// Returns false in case of error (memory allocation, invalid codes).
|
||||
static int HuffmanCodeLengthsToCodes(const int* const code_lengths,
|
||||
static int HuffmanCodeLengthsToCodes(const uint8_t* const code_lengths,
|
||||
int code_lengths_size,
|
||||
int* const huff_codes) {
|
||||
int symbol;
|
||||
@@ -133,35 +128,21 @@ static int HuffmanCodeLengthsToCodes(const int* const code_lengths,
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifndef USE_LUT_REVERSE_BITS
|
||||
|
||||
static int ReverseBitsShort(int bits, int num_bits) {
|
||||
int retval = 0;
|
||||
int i;
|
||||
assert(num_bits <= 8); // Not a hard requirement, just for coherency.
|
||||
for (i = 0; i < num_bits; ++i) {
|
||||
retval <<= 1;
|
||||
retval |= bits & 1;
|
||||
bits >>= 1;
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static const uint8_t kReversedBits[16] = { // Pre-reversed 4-bit values.
|
||||
0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
|
||||
0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf
|
||||
static const uint8_t kReverse7[128] = {
|
||||
0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120,
|
||||
4, 68, 36, 100, 20, 84, 52, 116, 12, 76, 44, 108, 28, 92, 60, 124,
|
||||
2, 66, 34, 98, 18, 82, 50, 114, 10, 74, 42, 106, 26, 90, 58, 122,
|
||||
6, 70, 38, 102, 22, 86, 54, 118, 14, 78, 46, 110, 30, 94, 62, 126,
|
||||
1, 65, 33, 97, 17, 81, 49, 113, 9, 73, 41, 105, 25, 89, 57, 121,
|
||||
5, 69, 37, 101, 21, 85, 53, 117, 13, 77, 45, 109, 29, 93, 61, 125,
|
||||
3, 67, 35, 99, 19, 83, 51, 115, 11, 75, 43, 107, 27, 91, 59, 123,
|
||||
7, 71, 39, 103, 23, 87, 55, 119, 15, 79, 47, 111, 31, 95, 63, 127
|
||||
};
|
||||
|
||||
static int ReverseBitsShort(int bits, int num_bits) {
|
||||
const uint8_t v = (kReversedBits[bits & 0xf] << 4) | kReversedBits[bits >> 4];
|
||||
assert(num_bits <= 8);
|
||||
return v >> (8 - num_bits);
|
||||
return kReverse7[bits] >> (7 - num_bits);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static int TreeAddSymbol(HuffmanTree* const tree,
|
||||
int symbol, int code, int code_length) {
|
||||
int step = HUFF_LUT_BITS;
|
||||
@@ -170,13 +151,14 @@ static int TreeAddSymbol(HuffmanTree* const tree,
|
||||
const HuffmanTreeNode* const max_node = tree->root_ + tree->max_nodes_;
|
||||
assert(symbol == (int16_t)symbol);
|
||||
if (code_length <= HUFF_LUT_BITS) {
|
||||
int i;
|
||||
int i = 1 << (HUFF_LUT_BITS - code_length);
|
||||
base_code = ReverseBitsShort(code, code_length);
|
||||
for (i = 0; i < (1 << (HUFF_LUT_BITS - code_length)); ++i) {
|
||||
do {
|
||||
--i;
|
||||
const int idx = base_code | (i << code_length);
|
||||
tree->lut_symbol_[idx] = (int16_t)symbol;
|
||||
tree->lut_bits_[idx] = code_length;
|
||||
}
|
||||
} while (i > 0);
|
||||
} else {
|
||||
base_code = ReverseBitsShort((code >> (code_length - HUFF_LUT_BITS)),
|
||||
HUFF_LUT_BITS);
|
||||
@@ -206,7 +188,7 @@ static int TreeAddSymbol(HuffmanTree* const tree,
|
||||
}
|
||||
|
||||
int BrotliHuffmanTreeBuildImplicit(HuffmanTree* const tree,
|
||||
const int* const code_lengths,
|
||||
const uint8_t* const code_lengths,
|
||||
int code_lengths_size) {
|
||||
int symbol;
|
||||
int num_symbols = 0;
|
||||
@@ -264,41 +246,6 @@ int BrotliHuffmanTreeBuildImplicit(HuffmanTree* const tree,
|
||||
}
|
||||
}
|
||||
|
||||
int BrotliHuffmanTreeBuildExplicit(HuffmanTree* const tree,
|
||||
const int* const code_lengths,
|
||||
const int* const codes,
|
||||
const int* const symbols,
|
||||
int max_symbol,
|
||||
int num_symbols) {
|
||||
int ok = 0;
|
||||
int i;
|
||||
|
||||
assert(tree != NULL);
|
||||
assert(code_lengths != NULL);
|
||||
assert(codes != NULL);
|
||||
assert(symbols != NULL);
|
||||
|
||||
// Initialize the tree. Will fail if num_symbols = 0.
|
||||
if (!TreeInit(tree, num_symbols)) return 0;
|
||||
|
||||
// Add symbols one-by-one.
|
||||
for (i = 0; i < num_symbols; ++i) {
|
||||
if (codes[i] != NON_EXISTENT_SYMBOL) {
|
||||
if (symbols[i] < 0 || symbols[i] >= max_symbol) {
|
||||
goto End;
|
||||
}
|
||||
if (!TreeAddSymbol(tree, symbols[i], codes[i], code_lengths[i])) {
|
||||
goto End;
|
||||
}
|
||||
}
|
||||
}
|
||||
ok = 1;
|
||||
End:
|
||||
ok = ok && IsFull(tree);
|
||||
if (!ok) BrotliHuffmanTreeRelease(tree);
|
||||
return ok;
|
||||
}
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
+1
-12
@@ -43,7 +43,6 @@ struct HuffmanTree {
|
||||
HuffmanTreeNode* root_; // all the nodes, starting at root.
|
||||
int max_nodes_; // max number of nodes
|
||||
int num_nodes_; // number of currently occupied nodes
|
||||
int fixed_bit_length_; // If non-zero, uses fixed length coding
|
||||
};
|
||||
|
||||
// Returns true if the given node is not a leaf of the Huffman tree.
|
||||
@@ -65,19 +64,9 @@ void BrotliHuffmanTreeRelease(HuffmanTree* const tree);
|
||||
// Builds Huffman tree assuming code lengths are implicitly in symbol order.
|
||||
// Returns false in case of error (invalid tree or memory error).
|
||||
int BrotliHuffmanTreeBuildImplicit(HuffmanTree* const tree,
|
||||
const int* const code_lengths,
|
||||
const uint8_t* const code_lengths,
|
||||
int code_lengths_size);
|
||||
|
||||
// Build a Huffman tree with explicitly given lists of code lengths, codes
|
||||
// and symbols. Verifies that all symbols added are smaller than max_symbol.
|
||||
// Returns false in case of an invalid symbol, invalid tree or memory error.
|
||||
int BrotliHuffmanTreeBuildExplicit(HuffmanTree* const tree,
|
||||
const int* const code_lengths,
|
||||
const int* const codes,
|
||||
const int* const symbols,
|
||||
int max_symbol,
|
||||
int num_symbols);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Copyright 2013 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Functions for streaming input and output.
|
||||
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include "./streams.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
int BrotliMemInputFunction(void* data, uint8_t* buf, size_t count) {
|
||||
BrotliMemInput* input = (BrotliMemInput*)data;
|
||||
if (input->pos > input->length) {
|
||||
return -1;
|
||||
}
|
||||
if (input->pos + count > input->length) {
|
||||
count = input->length - input->pos;
|
||||
}
|
||||
memcpy(buf, input->buffer + input->pos, count);
|
||||
input->pos += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
BrotliInput BrotliInitMemInput(const uint8_t* buffer, size_t length,
|
||||
BrotliMemInput* mem_input) {
|
||||
mem_input->buffer = buffer;
|
||||
mem_input->length = length;
|
||||
mem_input->pos = 0;
|
||||
BrotliInput input;
|
||||
input.cb_ = &BrotliMemInputFunction;
|
||||
input.data_ = mem_input;
|
||||
return input;
|
||||
}
|
||||
|
||||
int BrotliMemOutputFunction(void* data, const uint8_t* buf, size_t count) {
|
||||
BrotliMemOutput* output = (BrotliMemOutput*)data;
|
||||
if (output->pos + count > output->length) {
|
||||
return -1;
|
||||
}
|
||||
memcpy(output->buffer + output->pos, buf, count);
|
||||
output->pos += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
BrotliOutput BrotliInitMemOutput(uint8_t* buffer, size_t length,
|
||||
BrotliMemOutput* mem_output) {
|
||||
mem_output->buffer = buffer;
|
||||
mem_output->length = length;
|
||||
mem_output->pos = 0;
|
||||
BrotliOutput output;
|
||||
output.cb_ = &BrotliMemOutputFunction;
|
||||
output.data_ = mem_output;
|
||||
return output;
|
||||
}
|
||||
|
||||
int BrotliStdinInputFunction(void* data, uint8_t* buf, size_t count) {
|
||||
return read(STDIN_FILENO, buf, count);
|
||||
}
|
||||
|
||||
BrotliInput BrotliStdinInput() {
|
||||
BrotliInput in;
|
||||
in.cb_ = BrotliStdinInputFunction;
|
||||
in.data_ = NULL;
|
||||
return in;
|
||||
}
|
||||
|
||||
int BrotliStdoutOutputFunction(void* data, const uint8_t* buf, size_t count) {
|
||||
return write(STDOUT_FILENO, buf, count);
|
||||
}
|
||||
|
||||
BrotliOutput BrotliStdoutOutput() {
|
||||
BrotliOutput out;
|
||||
out.cb_ = BrotliStdoutOutputFunction;
|
||||
out.data_ = NULL;
|
||||
return out;
|
||||
}
|
||||
|
||||
int BrotliFileOutputFunction(void* data, const uint8_t* buf, size_t count) {
|
||||
return fwrite(buf, 1, count, (FILE*)data);
|
||||
}
|
||||
|
||||
BrotliOutput BrotliFileOutput(FILE* f) {
|
||||
BrotliOutput out;
|
||||
out.cb_ = BrotliFileOutputFunction;
|
||||
out.data_ = f;
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2013 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Functions for streaming input and output.
|
||||
|
||||
#ifndef BROTLI_DEC_STREAMS_H_
|
||||
#define BROTLI_DEC_STREAMS_H_
|
||||
|
||||
#include <stdio.h>
|
||||
#include "./types.h"
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// Function pointer type used to read len bytes into buf. Returns the
|
||||
// number of bytes read or -1 on error.
|
||||
typedef int (*BrotliInputFunction)(void* data, uint8_t* buf, size_t len);
|
||||
|
||||
// Input callback function with associated data.
|
||||
typedef struct {
|
||||
BrotliInputFunction cb_;
|
||||
void* data_;
|
||||
} BrotliInput;
|
||||
|
||||
// Reads len bytes into buf, using the in callback.
|
||||
static BROTLI_INLINE int BrotliRead(BrotliInput in, uint8_t* buf, size_t len) {
|
||||
return in.cb_(in.data_, buf, len);
|
||||
}
|
||||
|
||||
// Function pointer type used to write len bytes into buf. Returns the
|
||||
// number of bytes written or -1 on error.
|
||||
typedef int (*BrotliOutputFunction)(void* data, const uint8_t* buf, size_t len);
|
||||
|
||||
// Output callback function with associated data.
|
||||
typedef struct {
|
||||
BrotliOutputFunction cb_;
|
||||
void* data_;
|
||||
} BrotliOutput;
|
||||
|
||||
// Writes len bytes into buf, using the out callback.
|
||||
static BROTLI_INLINE int BrotliWrite(BrotliOutput out,
|
||||
const uint8_t* buf, size_t len) {
|
||||
return out.cb_(out.data_, buf, len);
|
||||
}
|
||||
|
||||
// Memory region with position.
|
||||
typedef struct {
|
||||
const uint8_t* buffer;
|
||||
size_t length;
|
||||
size_t pos;
|
||||
} BrotliMemInput;
|
||||
|
||||
// Input callback where *data is a BrotliMemInput struct.
|
||||
int BrotliMemInputFunction(void* data, uint8_t* buf, size_t count);
|
||||
|
||||
// Returns an input callback that wraps the given memory region.
|
||||
BrotliInput BrotliInitMemInput(const uint8_t* buffer, size_t length,
|
||||
BrotliMemInput* mem_input);
|
||||
|
||||
// Output buffer with position.
|
||||
typedef struct {
|
||||
uint8_t* buffer;
|
||||
size_t length;
|
||||
size_t pos;
|
||||
} BrotliMemOutput;
|
||||
|
||||
// Output callback where *data is a BrotliMemOutput struct.
|
||||
int BrotliMemOutputFunction(void* data, const uint8_t* buf, size_t count);
|
||||
|
||||
// Returns an output callback that wraps the given memory region.
|
||||
BrotliOutput BrotliInitMemOutput(uint8_t* buffer, size_t length,
|
||||
BrotliMemOutput* mem_output);
|
||||
|
||||
// Input callback that reads from standard input.
|
||||
int BrotliStdinInputFunction(void* data, uint8_t* buf, size_t count);
|
||||
BrotliInput BrotliStdinInput();
|
||||
|
||||
// Output callback that writes to standard output.
|
||||
int BrotliStdoutOutputFunction(void* data, const uint8_t* buf, size_t count);
|
||||
BrotliOutput BrotliStdoutOutput();
|
||||
|
||||
// Output callback that writes to a file.
|
||||
int BrotliFileOutputFunction(void* data, const uint8_t* buf, size_t count);
|
||||
BrotliOutput BrotliFileOutput(FILE* f);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // BROTLI_DEC_STREAMS_H_
|
||||
Reference in New Issue
Block a user