Updates to Brotli compression format, decoder and encoder

This commit contains a batch of changes that were made to the Brotli
compression algorithm in the last month. Most important changes:

   * Format change: don't push distances representing static dictionary words to the distance cache.
   * Fix decoder invalid memory access bug caused by building a non-complete Huffman tree.
   * Add a mode parameter to the encoder interface.
   * Use different hashers for text and font mode.
   * Add a heuristics to the hasher for skipping non-compressible data.
   * Exhaustive search of static dictionary during backward reference search.

brotli/enc/static_dict.h

@@ -0,0 +1,68 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Class to model the static dictionary.
+
+#ifndef BROTLI_ENC_STATIC_DICT_H_
+#define BROTLI_ENC_STATIC_DICT_H_
+
+#include <algorithm>
+#include <unordered_map>
+#include <string>
+
+namespace brotli {
+
+class StaticDictionary {
+ public:
+  StaticDictionary() {}
+  void Insert(const std::string &str, int len, int dist) {
+    int ix = (dist << 6) + len;
+    std::unordered_map<std::string, int>::const_iterator it = map_.find(str);
+    if (it != map_.end() && ix >= it->second) {
+      return;
+    }
+    map_[str] = ix;
+    int v = 0;
+    for (int i = 0; i < 4 && i < str.size(); ++i) {
+      v += str[i] << (8 * i);
+    }
+    if (prefix_map_[v] < str.size()) {
+      prefix_map_[v] = str.size();
+    }
+  }
+  int GetLength(int v) const {
+    std::unordered_map<int, int>::const_iterator it = prefix_map_.find(v);
+    if (it == prefix_map_.end()) {
+      return 0;
+    }
+    return it->second;
+  }
+  bool Get(const std::string &str, int *len, int *dist) const {
+    std::unordered_map<std::string, int>::const_iterator it = map_.find(str);
+    if (it == map_.end()) {
+      return false;
+    }
+    int v = it->second;
+    *len = v & 63;
+    *dist = v >> 6;
+    return true;
+  }
+ private:
+  std::unordered_map<std::string, int> map_;
+  std::unordered_map<int, int> prefix_map_;
+};
+
+}  // namespace brotli
+
+#endif  // BROTLI_ENC_STATIC_DICT_H_