zumun/woff2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates to Brotli compression format, decoder and encoder

Browse Source 
This commit contains a batch of changes that were made to the Brotli
compression algorithm in the last month. Most important changes:

   * Fixes to the spec.
   * Change of code length code order.
   * Use a 2-level Huffman lookup table in the decoder.
   * Faster uncompressed meta-block decoding.
   * Optimized encoding of the Huffman code.
   * Detection of UTF-8 input encoding.
   * UTF-8 based literal cost modeling for improved
     backward reference selection.
This commit is contained in:
Zoltan Szabadka
2014-02-14 15:04:23 +01:00
parent dfc5a9f215
commit cbd5cb55f4
16 changed files with 912 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files
brotli/enc/backward_references.cc
+105 -47
View File
@@ -45,66 +45,97 @@ void CreateBackwardReferences(size_t num_bytes,
average_cost /= num_bytes;
hasher->set_average_cost(average_cost);
// M1 match is for considering for two repeated copies, if moving
// one literal form the previous copy to the current one allows the
// current copy to be more efficient (because the way static dictionary
// codes words). M1 matching improves text compression density by ~0.15 %.
bool match_found_M1 = false;
size_t best_len_M1 = 0;
size_t best_len_code_M1 = 0;
size_t best_dist_M1 = 0;
double best_score_M1 = 0;
while (i + 2 < i_end) {
size_t best_len = 0;
size_t best_len_code = 0;
size_t best_dist = 0;
double best_score = 0;
size_t max_distance = std::min(i + i_diff, max_backward_limit);
bool in_dictionary;
hasher->set_insert_length(insert_length);
bool match_found = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff, i_end - i, max_distance,
&best_len, &best_len_code, &best_dist, &best_score);
&best_len, &best_len_code, &best_dist, &best_score, &in_dictionary);
bool best_in_dictionary = in_dictionary;
if (match_found) {
// Found a match. Let's look for something even better ahead.
int delayed_backward_references_in_row = 0;
while (i + 4 < i_end &&
delayed_backward_references_in_row < 4) {
size_t best_len_2 = 0;
size_t best_len_code_2 = 0;
size_t best_dist_2 = 0;
double best_score_2 = 0;
max_distance = std::min(i + i_diff + 1, max_backward_limit);
if (match_found_M1 && best_score_M1 > best_score) {
// Two copies after each other. Take the last literal from the
// last copy, and use it as the first of this one.
(commands->rbegin())->copy_length_ -= 1;
(commands->rbegin())->copy_length_code_ -= 1;
hasher->Store(ringbuffer + i, i + i_diff);
match_found = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff + 1, i_end - i - 1, max_distance,
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2);
double cost_diff_lazy = 0;
if (best_len >= 4) {
cost_diff_lazy +=
literal_cost[(i + 4) & ringbuffer_mask] - average_cost;
}
{
const int tail_length = best_len_2 - best_len + 1;
for (int k = 0; k < tail_length; ++k) {
cost_diff_lazy -=
literal_cost[(i + best_len + k) & ringbuffer_mask] -
average_cost;
--i;
best_len = best_len_M1;
best_len_code = best_len_code_M1;
best_dist = best_dist_M1;
best_score = best_score_M1;
// in_dictionary doesn't need to be correct, but it is the only
// reason why M1 matching should be beneficial here. Setting it here
// will only disable further M1 matching against this copy.
best_in_dictionary = true;
in_dictionary = true;
} else {
// Found a match. Let's look for something even better ahead.
int delayed_backward_references_in_row = 0;
while (i + 4 < i_end &&
delayed_backward_references_in_row < 4) {
size_t best_len_2 = 0;
size_t best_len_code_2 = 0;
size_t best_dist_2 = 0;
double best_score_2 = 0;
max_distance = std::min(i + i_diff + 1, max_backward_limit);
hasher->Store(ringbuffer + i, i + i_diff);
match_found = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff + 1, i_end - i - 1, max_distance,
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2,
&in_dictionary);
double cost_diff_lazy = 0;
if (best_len >= 4) {
cost_diff_lazy +=
literal_cost[(i + 4) & ringbuffer_mask] - average_cost;
}
}
// If we are not inserting any symbols, inserting one is more
// expensive than if we were inserting symbols anyways.
if (insert_length < 1) {
cost_diff_lazy += 0.97;
}
// Add bias to slightly avoid lazy matching.
cost_diff_lazy += 2.0 + delayed_backward_references_in_row * 0.2;
cost_diff_lazy += 0.04 * literal_cost[i & ringbuffer_mask];
{
const int tail_length = best_len_2 - best_len + 1;
for (int k = 0; k < tail_length; ++k) {
cost_diff_lazy -=
literal_cost[(i + best_len + k) & ringbuffer_mask] -
average_cost;
}
}
// If we are not inserting any symbols, inserting one is more
// expensive than if we were inserting symbols anyways.
if (insert_length < 1) {
cost_diff_lazy += 0.97;
}
// Add bias to slightly avoid lazy matching.
cost_diff_lazy += 2.0 + delayed_backward_references_in_row * 0.2;
cost_diff_lazy += 0.04 * literal_cost[i & ringbuffer_mask];
if (match_found && best_score_2 >= best_score + cost_diff_lazy) {
// Ok, let's just write one byte for now and start a match from the
// next byte.
++insert_length;
++delayed_backward_references_in_row;
best_len = best_len_2;
best_len_code = best_len_code_2;
best_dist = best_dist_2;
best_score = best_score_2;
i++;
} else {
break;
if (match_found && best_score_2 >= best_score + cost_diff_lazy) {
// Ok, let's just write one byte for now and start a match from the
// next byte.
++insert_length;
++delayed_backward_references_in_row;
best_len = best_len_2;
best_len_code = best_len_code_2;
best_dist = best_dist_2;
best_score = best_score_2;
best_in_dictionary = in_dictionary;
i++;
} else {
break;
}
}
}
Command cmd;
@@ -117,13 +148,40 @@ void CreateBackwardReferences(size_t num_bytes,
insert_length = 0;
++i;
for (int j = 1; j < best_len; ++j) {
// Copy all copied literals to the hasher, except the last one.
// We cannot store the last one yet, otherwise we couldn't find
// the possible M1 match.
for (int j = 1; j < best_len - 1; ++j) {
if (i + 2 < i_end) {
hasher->Store(ringbuffer + i, i + i_diff);
}
++i;
}
// Prepare M1 match.
if (best_len >= 4 && i + 20 < i_end && !best_in_dictionary) {
max_distance = std::min(i + i_diff, max_backward_limit);
match_found_M1 = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff, i_end - i, max_distance,
&best_len_M1, &best_len_code_M1, &best_dist_M1, &best_score_M1,
&in_dictionary);
} else {
match_found_M1 = false;
in_dictionary = false;
}
// This byte is just moved from the previous copy to the current,
// that is no gain.
best_score_M1 -= literal_cost[i & ringbuffer_mask];
// Adjust for losing the opportunity for lazy matching.
best_score_M1 -= 3.75;
// Store the last one of the match.
if (i + 2 < i_end) {
hasher->Store(ringbuffer + i, i + i_diff);
}
++i;
} else {
match_found_M1 = false;
++insert_length;
hasher->Store(ringbuffer + i, i + i_diff);
++i;
brotli/enc/bit_cost.h
+1 -1
View File
@@ -93,7 +93,7 @@ static inline int HuffmanBitCost(const uint8_t* depth, int length) {
cost[17] += 3;
int tree_size = 0;
int bits = 6 + 3 * max_depth; // huffman tree of huffman tree cost
int bits = 6 + 2 * max_depth; // huffman tree of huffman tree cost
for (int i = 0; i < kCodeLengthCodes; ++i) {
bits += histogram[i] * cost[i]; // huffman tree bit cost
tree_size += histogram[i];
brotli/enc/block_splitter.cc
+3 -3
View File
@@ -31,16 +31,16 @@
namespace brotli {
static const int kMaxLiteralHistograms = 48;
static const int kMaxLiteralHistograms = 100;
static const int kMaxCommandHistograms = 50;
static const double kLiteralBlockSwitchCost = 26;
static const double kCommandBlockSwitchCost = 13.5;
static const double kDistanceBlockSwitchCost = 14.6;
static const int kLiteralStrideLength = 70;
static const int kCommandStrideLength = 40;
static const int kSymbolsPerLiteralHistogram = 550;
static const int kSymbolsPerLiteralHistogram = 544;
static const int kSymbolsPerCommandHistogram = 530;
static const int kSymbolsPerDistanceHistogram = 550;
static const int kSymbolsPerDistanceHistogram = 544;
static const int kMinLengthForBlockSplitting = 128;
static const int kIterMulForRefining = 2;
static const int kMinItersForRefining = 100;
brotli/enc/encode.cc
+141 -48
View File
@@ -77,6 +77,68 @@ void EncodeVarLenUint8(int n, int* storage_ix, uint8_t* storage) {
}
}
int ParseAsUTF8(int* symbol, const uint8_t* input, int size) {
// ASCII
if ((input[0] & 0x80) == 0) {
*symbol = input[0];
if (*symbol > 0) {
return 1;
}
}
// 2-byte UTF8
if (size > 1 &&
(input[0] & 0xe0) == 0xc0 &&
(input[1] & 0xc0) == 0x80) {
*symbol = (((input[0] & 0x1f) << 6) |
(input[1] & 0x3f));
if (*symbol > 0x7f) {
return 2;
}
}
// 3-byte UFT8
if (size > 2 &&
(input[0] & 0xf0) == 0xe0 &&
(input[1] & 0xc0) == 0x80 &&
(input[2] & 0xc0) == 0x80) {
*symbol = (((input[0] & 0x0f) << 12) |
((input[1] & 0x3f) << 6) |
(input[2] & 0x3f));
if (*symbol > 0x7ff) {
return 3;
}
}
// 4-byte UFT8
if (size > 3 &&
(input[0] & 0xf8) == 0xf0 &&
(input[1] & 0xc0) == 0x80 &&
(input[2] & 0xc0) == 0x80 &&
(input[3] & 0xc0) == 0x80) {
*symbol = (((input[0] & 0x07) << 18) |
((input[1] & 0x3f) << 12) |
((input[2] & 0x3f) << 6) |
(input[3] & 0x3f));
if (*symbol > 0xffff && *symbol <= 0x10ffff) {
return 4;
}
}
// Not UTF8, emit a special symbol above the UTF8-code space
*symbol = 0x110000 | input[0];
return 1;
}
// Returns true if at least min_fraction of the data is UTF8-encoded.
bool IsMostlyUTF8(const uint8_t* data, size_t length, double min_fraction) {
size_t size_utf8 = 0;
size_t pos = 0;
while (pos < length) {
int symbol;
int bytes_read = ParseAsUTF8(&symbol, data + pos, length - pos);
pos += bytes_read;
if (symbol < 0x110000) size_utf8 += bytes_read;
}
return size_utf8 > min_fraction * length;
}
void EncodeMetaBlockLength(size_t meta_block_size,
bool is_last,
bool is_uncompressed,
@@ -118,7 +180,7 @@ void StoreHuffmanTreeOfHuffmanTreeToBitMask(
const uint8_t* code_length_bitdepth,
int* storage_ix, uint8_t* storage) {
static const uint8_t kStorageOrder[kCodeLengthCodes] = {
1, 2, 3, 4, 0, 17, 5, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
};
// Throw away trailing zeros:
int codes_to_store = kCodeLengthCodes;
@@ -147,7 +209,7 @@ void StoreHuffmanTreeOfHuffmanTreeToBitMask(
WriteBits(2, skip_some, storage_ix, storage);
for (int i = skip_some; i < codes_to_store; ++i) {
uint8_t len[] = { 2, 4, 3, 2, 2, 4 };
uint8_t bits[] = { 0, 5, 1, 3, 2, 13 };
uint8_t bits[] = { 0, 7, 3, 2, 1, 15 };
int v = code_length_bitdepth[kStorageOrder[i]];
WriteBits(len[v], bits[v], storage_ix, storage);
}
@@ -175,54 +237,49 @@ void StoreHuffmanTreeToBitMask(
}
template<int kSize>
void StoreHuffmanCode(const EntropyCode<kSize>& code, int alphabet_size,
int* storage_ix, uint8_t* storage) {
void StoreHuffmanCodeSimple(
const EntropyCode<kSize>& code, int alphabet_size,
int max_bits,
int* storage_ix, uint8_t* storage) {
const uint8_t *depth = &code.depth_[0];
int max_bits_counter = alphabet_size - 1;
int max_bits = 0;
while (max_bits_counter) {
max_bits_counter >>= 1;
++max_bits;
int symbols[4];
// Quadratic sort.
int k, j;
for (k = 0; k < code.count_; ++k) {
symbols[k] = code.symbols_[k];
}
if (code.count_ == 0) { // emit minimal tree for empty cases
// bits: small tree marker: 1, count-1: 0, max_bits-sized encoding for 0
WriteBits(4 + max_bits, 0x1, storage_ix, storage);
return;
}
if (code.count_ <= 4) {
int symbols[4];
// Quadratic sort.
int k, j;
for (k = 0; k < code.count_; ++k) {
symbols[k] = code.symbols_[k];
}
for (k = 0; k < code.count_; ++k) {
for (j = k + 1; j < code.count_; ++j) {
if (depth[symbols[j]] < depth[symbols[k]]) {
int t = symbols[k];
symbols[k] = symbols[j];
symbols[j] = t;
}
for (k = 0; k < code.count_; ++k) {
for (j = k + 1; j < code.count_; ++j) {
if (depth[symbols[j]] < depth[symbols[k]]) {
int t = symbols[k];
symbols[k] = symbols[j];
symbols[j] = t;
}
}
// Small tree marker to encode 1-4 symbols.
WriteBits(2, 1, storage_ix, storage);
WriteBits(2, code.count_ - 1, storage_ix, storage);
for (int i = 0; i < code.count_; ++i) {
WriteBits(max_bits, symbols[i], storage_ix, storage);
}
if (code.count_ == 4) {
if (depth[symbols[0]] == 2 &&
depth[symbols[1]] == 2 &&
depth[symbols[2]] == 2 &&
depth[symbols[3]] == 2) {
WriteBits(1, 0, storage_ix, storage);
} else {
WriteBits(1, 1, storage_ix, storage);
}
}
return;
}
// Small tree marker to encode 1-4 symbols.
WriteBits(2, 1, storage_ix, storage);
WriteBits(2, code.count_ - 1, storage_ix, storage);
for (int i = 0; i < code.count_; ++i) {
WriteBits(max_bits, symbols[i], storage_ix, storage);
}
if (code.count_ == 4) {
if (depth[symbols[0]] == 2 &&
depth[symbols[1]] == 2 &&
depth[symbols[2]] == 2 &&
depth[symbols[3]] == 2) {
WriteBits(1, 0, storage_ix, storage);
} else {
WriteBits(1, 1, storage_ix, storage);
}
}
}
template<int kSize>
void StoreHuffmanCodeComplex(
const EntropyCode<kSize>& code, int alphabet_size,
int* storage_ix, uint8_t* storage) {
const uint8_t *depth = &code.depth_[0];
uint8_t huffman_tree[kSize];
uint8_t huffman_tree_extra_bits[kSize];
int huffman_tree_size = 0;
@@ -246,6 +303,31 @@ void StoreHuffmanCode(const EntropyCode<kSize>& code, int alphabet_size,
storage_ix, storage);
}
template<int kSize>
void StoreHuffmanCode(const EntropyCode<kSize>& code, int alphabet_size,
int* storage_ix, uint8_t* storage) {
int max_bits_counter = alphabet_size - 1;
int max_bits = 0;
while (max_bits_counter) {
max_bits_counter >>= 1;
++max_bits;
}
if (code.count_ == 0) {
// Emit a minimal tree for empty cases.
// bits: small tree marker: 1, count-1: 0, max_bits-sized encoding for 0
WriteBits(4 + max_bits, 0x1, storage_ix, storage);
} else if (code.count_ <= 4) {
StoreHuffmanCodeSimple(
code, alphabet_size, max_bits,
storage_ix, storage);
} else {
StoreHuffmanCodeComplex(
code, alphabet_size,
storage_ix, storage);
}
}
template<int kSize>
void StoreHuffmanCodes(const std::vector<EntropyCode<kSize> >& codes,
int alphabet_size,
@@ -798,12 +880,23 @@ void BrotliCompressor::WriteMetaBlock(const size_t input_size,
const bool is_last,
size_t* encoded_size,
uint8_t* encoded_buffer) {
static const double kMinUTF8Ratio = 0.75;
bool utf8_mode = false;
std::vector<Command> commands;
if (input_size > 0) {
ringbuffer_.Write(input_buffer, input_size);
EstimateBitCostsForLiterals(input_pos_, input_size,
kRingBufferMask, ringbuffer_.start(),
&literal_cost_[0]);
utf8_mode = IsMostlyUTF8(
&ringbuffer_.start()[input_pos_ & kRingBufferMask],
input_size, kMinUTF8Ratio);
if (utf8_mode) {
EstimateBitCostsForLiteralsUTF8(input_pos_, input_size,
kRingBufferMask, ringbuffer_.start(),
&literal_cost_[0]);
} else {
EstimateBitCostsForLiterals(input_pos_, input_size,
kRingBufferMask, ringbuffer_.start(),
&literal_cost_[0]);
}
CreateBackwardReferences(input_size, input_pos_,
ringbuffer_.start(),
&literal_cost_[0],
brotli/enc/entropy_encode.cc
+107 -12
View File
@@ -182,6 +182,12 @@ void WriteHuffmanTreeRepetitions(
++(*tree_size);
--repetitions;
}
if (repetitions == 7) {
tree[*tree_size] = value;
extra_bits[*tree_size] = 0;
++(*tree_size);
--repetitions;
}
if (repetitions < 3) {
for (int i = 0; i < repetitions; ++i) {
tree[*tree_size] = value;
@@ -208,6 +214,12 @@ void WriteHuffmanTreeRepetitionsZeros(
uint8_t* tree,
uint8_t* extra_bits,
int* tree_size) {
if (repetitions == 11) {
tree[*tree_size] = 0;
extra_bits[*tree_size] = 0;
++(*tree_size);
--repetitions;
}
if (repetitions < 3) {
for (int i = 0; i < repetitions; ++i) {
tree[*tree_size] = 0;
@@ -230,11 +242,6 @@ void WriteHuffmanTreeRepetitionsZeros(
}
// Heuristics for selecting the stride ranges to collapse.
int ValuesShouldBeCollapsedToStrideAverage(int a, int b) {
return abs(a - b) < 4;
}
int OptimizeHuffmanCountsForRle(int length, int* counts) {
int stride;
int limit;
@@ -251,6 +258,35 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
break;
}
}
{
int nonzeros = 0;
int smallest_nonzero = 1 << 30;
for (i = 0; i < length; ++i) {
if (counts[i] != 0) {
++nonzeros;
if (smallest_nonzero > counts[i]) {
smallest_nonzero = counts[i];
}
}
}
if (nonzeros < 5) {
// Small histogram will model it well.
return 1;
}
int zeros = length - nonzeros;
if (smallest_nonzero < 4) {
if (zeros < 6) {
for (i = 1; i < length - 1; ++i) {
if (counts[i - 1] != 0 && counts[i] == 0 && counts[i + 1] != 0) {
counts[i] = 1;
}
}
}
}
if (nonzeros < 28) {
return 1;
}
}
// 2) Let's mark all population counts that already can be encoded
// with an rle code.
good_for_rle = (uint8_t*)calloc(length, 1);
@@ -282,13 +318,15 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
}
}
// 3) Let's replace those population counts that lead to more rle codes.
// Math here is in 24.8 fixed point representation.
const int streak_limit = 1240;
stride = 0;
limit = (counts[0] + counts[1] + counts[2]) / 3 + 1;
limit = 256 * (counts[0] + counts[1] + counts[2]) / 3 + 420;
sum = 0;
for (i = 0; i < length + 1; ++i) {
if (i == length || good_for_rle[i] ||
(i != 0 && good_for_rle[i - 1]) ||
!ValuesShouldBeCollapsedToStrideAverage(counts[i], limit)) {
abs(256 * counts[i] - limit) >= streak_limit) {
if (stride >= 4 || (stride >= 3 && sum == 0)) {
int k;
// The stride must end, collapse what we have, if we have enough (4).
@@ -311,9 +349,9 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
if (i < length - 2) {
// All interesting strides have a count of at least 4,
// at least when non-zeros.
limit = (counts[i] + counts[i + 1] + counts[i + 2]) / 3 + 1;
limit = 256 * (counts[i] + counts[i + 1] + counts[i + 2]) / 3 + 420;
} else if (i < length) {
limit = counts[i];
limit = 256 * counts[i];
} else {
limit = 0;
}
@@ -322,7 +360,10 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
if (i != length) {
sum += counts[i];
if (stride >= 4) {
limit = (sum + stride / 2) / stride;
limit = (256 * sum + stride / 2) / stride;
}
if (stride == 4) {
limit += 120;
}
}
}
@@ -331,16 +372,70 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
}
static void DecideOverRleUse(const uint8_t* depth, const int length,
bool *use_rle_for_non_zero,
bool *use_rle_for_zero) {
int total_reps_zero = 0;
int total_reps_non_zero = 0;
int count_reps_zero = 0;
int count_reps_non_zero = 0;
int new_length = length;
for (int i = 0; i < length; ++i) {
if (depth[length - i - 1] == 0) {
--new_length;
} else {
break;
}
}
for (uint32_t i = 0; i < new_length;) {
const int value = depth[i];
int reps = 1;
// Find rle coding for longer codes.
// Shorter codes seem not to benefit from rle.
for (uint32_t k = i + 1; k < new_length && depth[k] == value; ++k) {
++reps;
}
if (reps >= 3 && value == 0) {
total_reps_zero += reps;
++count_reps_zero;
}
if (reps >= 4 && value != 0) {
total_reps_non_zero += reps;
++count_reps_non_zero;
}
i += reps;
}
total_reps_non_zero -= count_reps_non_zero * 2;
total_reps_zero -= count_reps_zero * 2;
*use_rle_for_non_zero = total_reps_non_zero > 2;
*use_rle_for_zero = total_reps_zero > 2;
}
void WriteHuffmanTree(const uint8_t* depth, const int length,
uint8_t* tree,
uint8_t* extra_bits_data,
int* huffman_tree_size) {
int previous_value = 8;
// First gather statistics on if it is a good idea to do rle.
bool use_rle_for_non_zero;
bool use_rle_for_zero;
DecideOverRleUse(depth, length, &use_rle_for_non_zero, &use_rle_for_zero);
// Actual rle coding.
for (uint32_t i = 0; i < length;) {
const int value = depth[i];
int reps = 1;
for (uint32_t k = i + 1; k < length && depth[k] == value; ++k) {
++reps;
if (length > 50) {
// Find rle coding for longer codes.
// Shorter codes seem not to benefit from rle.
if ((value != 0 && use_rle_for_non_zero) ||
(value == 0 && use_rle_for_zero)) {
for (uint32_t k = i + 1; k < length && depth[k] == value; ++k) {
++reps;
}
}
}
if (value == 0) {
WriteHuffmanTreeRepetitionsZeros(reps, tree, extra_bits_data,
brotli/enc/entropy_encode.h
+1 -1
View File
@@ -86,7 +86,7 @@ void BuildEntropyCode(const Histogram<kSize>& histogram,
++code->count_;
}
}
if (code->count_ >= 64) {
if (alphabet_size >= 50 && code->count_ >= 16) {
int counts[kSize];
memcpy(counts, &histogram.data_[0], sizeof(counts[0]) * kSize);
OptimizeHuffmanCountsForRle(alphabet_size, counts);
brotli/enc/hash.h
+9 -4
View File
@@ -150,7 +150,10 @@ class HashLongestMatch {
size_t * __restrict best_len_out,
size_t * __restrict best_len_code_out,
size_t * __restrict best_distance_out,
double * __restrict best_score_out) {
double * __restrict best_score_out,
bool * __restrict in_dictionary) {
*in_dictionary = true;
*best_len_code_out = 0;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
const double start_cost4 = literal_cost == NULL ? 20 :
literal_cost[cur_ix_masked] +
@@ -166,9 +169,9 @@ class HashLongestMatch {
literal_cost[(cur_ix + 1) & ring_buffer_mask] + 1.2;
bool match_found = false;
// Don't accept a short copy from far away.
double best_score = 8.25;
double best_score = 8.11;
if (insert_length_ < 4) {
double cost_diff[4] = { 0.20, 0.09, 0.05, 0.03 };
double cost_diff[4] = { 0.10, 0.04, 0.02, 0.01 };
best_score += cost_diff[insert_length_];
}
size_t best_len = *best_len_out;
@@ -235,6 +238,7 @@ class HashLongestMatch {
*best_distance_out = best_ix;
*best_score_out = best_score;
match_found = true;
*in_dictionary = backward > max_backward;
}
}
}
@@ -257,7 +261,7 @@ class HashLongestMatch {
continue;
}
int len = 2;
const double score = start_cost2 - 1.70 * Log2Floor(backward);
const double score = start_cost2 - 2.3 * Log2Floor(backward);
if (best_score < score) {
best_score = score;
@@ -309,6 +313,7 @@ class HashLongestMatch {
*best_distance_out = best_ix;
*best_score_out = best_score;
match_found = true;
*in_dictionary = false;
}
}
}
brotli/enc/literal_cost.cc
+99
View File
@@ -22,6 +22,104 @@
namespace brotli {
static int UTF8Position(int last, int c, int clamp) {
if (c < 128) {
return 0; // Next one is the 'Byte 1' again.
} else if (c >= 192) {
return std::min(1, clamp); // Next one is the 'Byte 2' of utf-8 encoding.
} else {
// Let's decide over the last byte if this ends the sequence.
if (last < 0xe0) {
return 0; // Completed two or three byte coding.
} else {
return std::min(2, clamp); // Next one is the 'Byte 3' of utf-8 encoding.
}
}
}
static int DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
const uint8_t *data) {
int counts[3] = { 0 };
int max_utf8 = 1; // should be 2, but 1 compresses better.
int last_c = 0;
int utf8_pos = 0;
for (int i = 0; i < len; ++i) {
int c = data[(pos + i) & mask];
utf8_pos = UTF8Position(last_c, c, 2);
++counts[utf8_pos];
last_c = c;
}
if (counts[2] < 500) {
max_utf8 = 1;
}
if (counts[1] + counts[2] < 25) {
max_utf8 = 0;
}
return max_utf8;
}
void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
// max_utf8 is 0 (normal ascii single byte modeling),
// 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling).
const int max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
int histogram[3][256] = { { 0 } };
int window_half = 495;
int in_window = std::min(static_cast<size_t>(window_half), len);
int in_window_utf8[3] = { 0 };
// Bootstrap histograms.
int last_c = 0;
int utf8_pos = 0;
for (int i = 0; i < in_window; ++i) {
int c = data[(pos + i) & mask];
++histogram[utf8_pos][c];
++in_window_utf8[utf8_pos];
utf8_pos = UTF8Position(last_c, c, max_utf8);
last_c = c;
}
// Compute bit costs with sliding window.
for (int i = 0; i < len; ++i) {
if (i - window_half >= 0) {
// Remove a byte in the past.
int c = (i - window_half - 1) < 0 ?
0 : data[(pos + i - window_half - 1) & mask];
int last_c = (i - window_half - 2) < 0 ?
0 : data[(pos + i - window_half - 2) & mask];
int utf8_pos2 = UTF8Position(last_c, c, max_utf8);
--histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
--in_window_utf8[utf8_pos2];
}
if (i + window_half < len) {
// Add a byte in the future.
int c = (i + window_half - 1) < 0 ?
0 : data[(pos + i + window_half - 1) & mask];
int last_c = (i + window_half - 2) < 0 ?
0 : data[(pos + i + window_half - 2) & mask];
int utf8_pos2 = UTF8Position(last_c, c, max_utf8);
++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
++in_window_utf8[utf8_pos2];
}
int c = i < 1 ? 0 : data[(pos + i - 1) & mask];
int last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
int utf8_pos = UTF8Position(last_c, c, max_utf8);
int masked_pos = (pos + i) & mask;
int histo = histogram[utf8_pos][data[masked_pos]];
if (histo == 0) {
histo = 1;
}
cost[masked_pos] = log2(static_cast<double>(in_window_utf8[utf8_pos])
/ histo);
cost[masked_pos] += 0.02905;
if (cost[masked_pos] < 1.0) {
cost[masked_pos] *= 0.5;
cost[masked_pos] += 0.5;
}
}
}
void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
int histogram[256] = { 0 };
@@ -59,4 +157,5 @@ void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
}
}
} // namespace brotli
brotli/enc/literal_cost.h
+6 -1
View File
@@ -26,7 +26,12 @@ namespace brotli {
// ringbuffer (data, mask) will take entropy coded and writes these estimates
// to the ringbuffer (cost, mask).
void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost);
const uint8_t *data,
float *cost);
void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
const uint8_t *data,
float *cost);
} // namespace brotli