Updates to Brotli compression format, decoder and encoder

This commit contains a batch of changes that were made to the Brotli
compression algorithm in the last three weeks. Most important changes:

  * Added UTF8 context model for good text compression.
  * Simplified context modeling by having only 4 context modes.
  * Per-block context mode selection.
  * Faster backward copying and bit reading functions.
  * More efficient histogram coding.
  * Streaming support for the decoder and encoder.

brotli/enc/histogram.cc

@@ -31,10 +31,10 @@ void BuildHistograms(
     const BlockSplit& literal_split,
     const BlockSplit& insert_and_copy_split,
     const BlockSplit& dist_split,
-    const uint8_t* input_buffer,
+    const uint8_t* ringbuffer,
     size_t pos,
-    int context_mode,
-    int distance_context_mode,
+    size_t mask,
+    const std::vector<int>& context_modes,
     std::vector<HistogramLiteral>* literal_histograms,
     std::vector<HistogramCommand>* insert_and_copy_histograms,
     std::vector<HistogramDistance>* copy_dist_histograms) {
@@ -48,25 +48,47 @@ void BuildHistograms(
         cmd.command_prefix_);
     for (int j = 0; j < cmd.insert_length_; ++j) {
       literal_it.Next();
-      uint8_t prev_byte = pos > 0 ? input_buffer[pos - 1] : 0;
-      uint8_t prev_byte2 = pos > 1 ? input_buffer[pos - 2] : 0;
-      uint8_t prev_byte3 = pos > 2 ? input_buffer[pos - 3] : 0;
-      int context = (literal_it.type_ * NumContexts(context_mode) +
-                     Context(prev_byte, prev_byte2, prev_byte3, context_mode));
-      (*literal_histograms)[context].Add(input_buffer[pos]);
+      uint8_t prev_byte = pos > 0 ? ringbuffer[(pos - 1) & mask] : 0;
+      uint8_t prev_byte2 = pos > 1 ? ringbuffer[(pos - 2) & mask] : 0;
+      int context = (literal_it.type_ << kLiteralContextBits) +
+          Context(prev_byte, prev_byte2, context_modes[literal_it.type_]);
+      (*literal_histograms)[context].Add(ringbuffer[pos & mask]);
       ++pos;
     }
     pos += cmd.copy_length_;
     if (cmd.copy_length_ > 0 && cmd.distance_prefix_ != 0xffff) {
       dist_it.Next();
-      int context = dist_it.type_;
-      if (distance_context_mode > 0) {
-        context <<= 2;
-        context += (cmd.copy_length_ > 4) ? 3 : cmd.copy_length_ - 2;
-      }
+      int context = (dist_it.type_ << kDistanceContextBits) +
+          ((cmd.copy_length_ > 4) ? 3 : cmd.copy_length_ - 2);
       (*copy_dist_histograms)[context].Add(cmd.distance_prefix_);
     }
   }
 }
 
+void BuildLiteralHistogramsForBlockType(
+    const std::vector<Command>& cmds,
+    const BlockSplit& literal_split,
+    const uint8_t* ringbuffer,
+    size_t pos,
+    size_t mask,
+    int block_type,
+    int context_mode,
+    std::vector<HistogramLiteral>* histograms) {
+  BlockSplitIterator literal_it(literal_split);
+  for (int i = 0; i < cmds.size(); ++i) {
+    const Command &cmd = cmds[i];
+    for (int j = 0; j < cmd.insert_length_; ++j) {
+      literal_it.Next();
+      if (literal_it.type_ == block_type) {
+        uint8_t prev_byte = pos > 0 ? ringbuffer[(pos - 1) & mask] : 0;
+        uint8_t prev_byte2 = pos > 1 ? ringbuffer[(pos - 2) & mask] : 0;
+        int context = Context(prev_byte, prev_byte2, context_mode);
+        (*histograms)[context].Add(ringbuffer[pos & mask]);
+      }
+      ++pos;
+    }
+    pos += cmd.copy_length_;
+  }
+}
+
 }  // namespace brotli