Updates to Brotli compression format, decoder and encoder
This commit contains a batch of changes that were made to the Brotli compression algorithm in the last three weeks. Most important changes: * Added UTF8 context model for good text compression. * Simplified context modeling by having only 4 context modes. * Per-block context mode selection. * Faster backward copying and bit reading functions. * More efficient histogram coding. * Streaming support for the decoder and encoder.
This commit is contained in:
Zoltan Szabadka
+127
-72
@@ -21,25 +21,124 @@
|
||||
|
||||
namespace brotli {
|
||||
|
||||
static const int kSigned2BitContextLookup[] = {
|
||||
// Second-order context lookup table for UTF8 byte streams.
|
||||
//
|
||||
// If p1 and p2 are the previous two bytes, we calcualte the context as
|
||||
//
|
||||
// context = kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256].
|
||||
//
|
||||
// If the previous two bytes are ASCII characters (i.e. < 128), this will be
|
||||
// equivalent to
|
||||
//
|
||||
// context = 4 * context1(p1) + context2(p2),
|
||||
//
|
||||
// where context1 is based on the previous byte in the following way:
|
||||
//
|
||||
// 0 : non-ASCII control
|
||||
// 1 : \t, \n, \r
|
||||
// 2 : space
|
||||
// 3 : other punctuation
|
||||
// 4 : " '
|
||||
// 5 : %
|
||||
// 6 : ( < [ {
|
||||
// 7 : ) > ] }
|
||||
// 8 : , ; :
|
||||
// 9 : .
|
||||
// 10 : =
|
||||
// 11 : number
|
||||
// 12 : upper-case vowel
|
||||
// 13 : upper-case consonant
|
||||
// 14 : lower-case vowel
|
||||
// 15 : lower-case consonant
|
||||
//
|
||||
// and context2 is based on the second last byte:
|
||||
//
|
||||
// 0 : control, space
|
||||
// 1 : punctuation
|
||||
// 2 : upper-case letter, number
|
||||
// 3 : lower-case letter
|
||||
//
|
||||
// If the last byte is ASCII, and the second last byte is not (in a valid UTF8
|
||||
// stream it will be a continuation byte, value between 128 and 191), the
|
||||
// context is the same as if the second last byte was an ASCII control or space.
|
||||
//
|
||||
// If the last byte is a UTF8 lead byte (value >= 192), then the next byte will
|
||||
// be a continuation byte and the context id is 2 or 3 depending on the LSB of
|
||||
// the last byte and to a lesser extent on the second last byte if it is ASCII.
|
||||
//
|
||||
// If the last byte is a UTF8 continuation byte, the second last byte can be:
|
||||
// - continuation byte: the next byte is probably ASCII or lead byte (assuming
|
||||
// 4-byte UTF8 characters are rare) and the context id is 0 or 1.
|
||||
// - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1
|
||||
// - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3
|
||||
//
|
||||
// The possible value combinations of the previous two bytes, the range of
|
||||
// context ids and the type of the next byte is summarized in the table below:
|
||||
//
|
||||
// |--------\-----------------------------------------------------------------|
|
||||
// | \ Last byte |
|
||||
// | Second \---------------------------------------------------------------|
|
||||
// | last byte \ ASCII | cont. byte | lead byte |
|
||||
// | \ (0-127) | (128-191) | (192-) |
|
||||
// |=============|===================|=====================|==================|
|
||||
// | ASCII | next: ASCII/lead | not valid | next: cont. |
|
||||
// | (0-127) | context: 4 - 63 | | context: 2 - 3 |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |
|
||||
// | (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | lead byte | not valid | next: ASCII/lead | not valid |
|
||||
// | (192-207) | | context: 0 - 1 | |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
// | lead byte | not valid | next: cont. | not valid |
|
||||
// | (208-) | | context: 2 - 3 | |
|
||||
// |-------------|-------------------|---------------------|------------------|
|
||||
static const uint8_t kUTF8ContextLookup[512] = {
|
||||
// Last byte.
|
||||
//
|
||||
// ASCII range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
|
||||
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
|
||||
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
|
||||
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
|
||||
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
|
||||
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
|
||||
// UTF8 continuation byte range.
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
|
||||
// UTF8 lead byte range.
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
|
||||
// Second last byte.
|
||||
//
|
||||
// ASCII range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
||||
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
|
||||
// UTF8 continuation byte range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
// UTF8 lead byte range.
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
|
||||
};
|
||||
|
||||
// Context lookup table for small signed integers.
|
||||
static const int kSigned3BitContextLookup[] = {
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
@@ -59,69 +158,25 @@ static const int kSigned3BitContextLookup[] = {
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
|
||||
};
|
||||
|
||||
static const int kSigned4BitContextLookup[] = {
|
||||
0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
||||
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15,
|
||||
};
|
||||
|
||||
enum ContextType {
|
||||
CONTEXT_NONE = 0,
|
||||
CONTEXT_FULL = 1,
|
||||
CONTEXT_MSB7 = 2,
|
||||
CONTEXT_MSB6 = 3,
|
||||
CONTEXT_MSB5 = 4,
|
||||
CONTEXT_MSB4 = 5,
|
||||
CONTEXT_MSB3 = 6,
|
||||
CONTEXT_MSB2 = 7,
|
||||
CONTEXT_MSB1 = 8,
|
||||
CONTEXT_IS_ZERO = 9,
|
||||
CONTEXT_SIGNED_2BIT = 10,
|
||||
CONTEXT_SIGNED_3BIT = 11,
|
||||
CONTEXT_SIGNED_4BIT = 12,
|
||||
CONTEXT_SIGNED_MIXED_3BYTE = 13,
|
||||
CONTEXT_LSB6 = 0,
|
||||
CONTEXT_MSB6 = 1,
|
||||
CONTEXT_UTF8 = 2,
|
||||
CONTEXT_SIGNED = 3
|
||||
};
|
||||
|
||||
static const int kContextSize[] = {
|
||||
1, 256, 128, 64, 32, 16, 8, 4, 2, 2, 4, 8, 16, 64,
|
||||
};
|
||||
|
||||
static inline int NumContexts(int mode) {
|
||||
return kContextSize[mode];
|
||||
}
|
||||
|
||||
static inline uint8_t Context(uint8_t prev_byte, uint8_t prev_byte2,
|
||||
uint8_t prev_byte3, int mode) {
|
||||
static inline uint8_t Context(uint8_t p1, uint8_t p2, int mode) {
|
||||
switch (mode) {
|
||||
case CONTEXT_NONE:
|
||||
return 0;
|
||||
case CONTEXT_IS_ZERO:
|
||||
return prev_byte == 0 ? 0 : 1;
|
||||
case CONTEXT_SIGNED_2BIT:
|
||||
return kSigned2BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_3BIT:
|
||||
return kSigned3BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_4BIT:
|
||||
return kSigned4BitContextLookup[prev_byte];
|
||||
case CONTEXT_SIGNED_MIXED_3BYTE:
|
||||
return ((kSigned3BitContextLookup[prev_byte] << 3) +
|
||||
(kSigned2BitContextLookup[prev_byte2] << 1) +
|
||||
(prev_byte3 == 0 ? 0 : 1));
|
||||
case CONTEXT_LSB6:
|
||||
return p1 & 0x3f;
|
||||
case CONTEXT_MSB6:
|
||||
return p1 >> 2;
|
||||
case CONTEXT_UTF8:
|
||||
return kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256];
|
||||
case CONTEXT_SIGNED:
|
||||
return (kSigned3BitContextLookup[p1] << 3) + kSigned3BitContextLookup[p2];
|
||||
default:
|
||||
return prev_byte >> (mode - 1);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user