Updates to Brotli compression format, decoder and encoder

This commit contains a batch of changes that were made to the Brotli
compression algorithm in the last three weeks. Most important changes:

  * Added UTF8 context model for good text compression.
  * Simplified context modeling by having only 4 context modes.
  * Per-block context mode selection.
  * Faster backward copying and bit reading functions.
  * More efficient histogram coding.
  * Streaming support for the decoder and encoder.

brotli/enc/literal_cost.cc

@@ -22,37 +22,39 @@
 
 namespace brotli {
 
-void EstimateBitCostsForLiterals(size_t len, const uint8_t *data, float *cost) {
+void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
+                                 const uint8_t *data, float *cost) {
   int histogram[256] = { 0 };
   int window_half = 2000;
   int in_window = std::min(static_cast<size_t>(window_half), len);
 
   // Bootstrap histogram.
   for (int i = 0; i < in_window; ++i) {
-    ++histogram[data[i]];
+    ++histogram[data[(pos + i) & mask]];
   }
 
   // Compute bit costs with sliding window.
   for (int i = 0; i < len; ++i) {
     if (i - window_half >= 0) {
       // Remove a byte in the past.
-      --histogram[data[i - window_half]];
+      --histogram[data[(pos + i - window_half) & mask]];
       --in_window;
     }
     if (i + window_half < len) {
       // Add a byte in the future.
-      ++histogram[data[i + window_half]];
+      ++histogram[data[(pos + i + window_half) & mask]];
       ++in_window;
     }
-    int histo = histogram[data[i]];
+    int masked_pos = (pos + i) & mask;
+    int histo = histogram[data[masked_pos]];
     if (histo == 0) {
       histo = 1;
     }
-    cost[i] = log2(static_cast<double>(in_window) / histo);
-    cost[i] += 0.03;
-    if (cost[i] < 1.0) {
-      cost[i] *= 0.5;
-      cost[i] += 0.5;
+    cost[masked_pos] = log2(static_cast<double>(in_window) / histo);
+    cost[masked_pos] += 0.03;
+    if (cost[masked_pos] < 1.0) {
+      cost[masked_pos] *= 0.5;
+      cost[masked_pos] += 0.5;
     }
   }
 }