zumun/woff2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates to Brotli compression format, decoder and encoder

Browse Source 
This commit contains a batch of changes that were made to the Brotli
compression algorithm in the last month. Most important changes:

   * Format change: don't push distances representing static dictionary words to the distance cache.
   * Fix decoder invalid memory access bug caused by building a non-complete Huffman tree.
   * Add a mode parameter to the encoder interface.
   * Use different hashers for text and font mode.
   * Add a heuristics to the hasher for skipping non-compressible data.
   * Exhaustive search of static dictionary during backward reference search.
This commit is contained in:
Zoltan Szabadka
2014-03-20 14:32:35 +01:00
parent 7f848593bd
commit 278b89484f
14 changed files with 546 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
brotli/enc/literal_cost.cc
+18 -15
View File
@@ -59,7 +59,8 @@ static int DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
}
void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
size_t cost_mask, const uint8_t *data,
float *cost) {
// max_utf8 is 0 (normal ascii single byte modeling),
// 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling).
@@ -110,18 +111,20 @@ void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
if (histo == 0) {
histo = 1;
}
cost[masked_pos] = log2(static_cast<double>(in_window_utf8[utf8_pos])
/ histo);
cost[masked_pos] += 0.02905;
if (cost[masked_pos] < 1.0) {
cost[masked_pos] *= 0.5;
cost[masked_pos] += 0.5;
float lit_cost = log2(static_cast<double>(in_window_utf8[utf8_pos])
/ histo);
lit_cost += 0.02905;
if (lit_cost < 1.0) {
lit_cost *= 0.5;
lit_cost += 0.5;
}
cost[(pos + i) & cost_mask] = lit_cost;
}
}
void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
size_t cost_mask, const uint8_t *data,
float *cost) {
int histogram[256] = { 0 };
int window_half = 2000;
int in_window = std::min(static_cast<size_t>(window_half), len);
@@ -143,17 +146,17 @@ void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
++histogram[data[(pos + i + window_half) & mask]];
++in_window;
}
int masked_pos = (pos + i) & mask;
int histo = histogram[data[masked_pos]];
int histo = histogram[data[(pos + i) & mask]];
if (histo == 0) {
histo = 1;
}
cost[masked_pos] = log2(static_cast<double>(in_window) / histo);
cost[masked_pos] += 0.029;
if (cost[masked_pos] < 1.0) {
cost[masked_pos] *= 0.5;
cost[masked_pos] += 0.5;
float lit_cost = log2(static_cast<double>(in_window) / histo);
lit_cost += 0.029;
if (lit_cost < 1.0) {
lit_cost *= 0.5;
lit_cost += 0.5;
}
cost[(pos + i) & cost_mask] = lit_cost;
}
}