zumun/woff2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Brotli compress/decompress utilities and makefiles

Browse Source
This commit is contained in:
Roderick Sheeter
2013-11-19 14:32:56 -08:00
parent 1571db36a9
commit 437bbad370
29 changed files with 3267 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
brotli/enc/Makefile
+11
View File
@@ -0,0 +1,11 @@
#brotli/enc
include ../../shared.mk
OBJS = backward_references.o block_splitter.o encode.o entropy_encode.o histogram.o literal_cost.o prefix.o
all : $(OBJS)
clean :
rm -f $(OBJS) $(SO)
brotli/enc/backward_references.cc
+8 -3
View File
@@ -47,27 +47,30 @@ void CreateBackwardReferences(size_t num_bytes,
while (i + 2 < i_end) {
size_t best_len = 0;
size_t best_len_code = 0;
size_t best_dist = 0;
double best_score = 0;
const size_t max_distance = std::min(i + i_diff, max_backward_limit);
size_t max_distance = std::min(i + i_diff, max_backward_limit);
hasher->set_insert_length(insert_length);
bool match_found = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff, i_end - i, max_distance,
&best_len, &best_dist, &best_score);
&best_len, &best_len_code, &best_dist, &best_score);
if (match_found) {
// Found a match. Let's look for something even better ahead.
int delayed_backward_references_in_row = 0;
while (i + 4 < i_end &&
delayed_backward_references_in_row < 4) {
size_t best_len_2 = 0;
size_t best_len_code_2 = 0;
size_t best_dist_2 = 0;
double best_score_2 = 0;
max_distance = std::min(i + i_diff + 1, max_backward_limit);
hasher->Store(ringbuffer + i, i + i_diff);
match_found = hasher->FindLongestMatch(
ringbuffer, literal_cost, ringbuffer_mask,
i + i_diff + 1, i_end - i - 1, max_distance,
&best_len_2, &best_dist_2, &best_score_2);
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2);
double cost_diff_lazy = 0;
if (best_len >= 4) {
cost_diff_lazy +=
@@ -96,6 +99,7 @@ void CreateBackwardReferences(size_t num_bytes,
++insert_length;
++delayed_backward_references_in_row;
best_len = best_len_2;
best_len_code = best_len_code_2;
best_dist = best_dist_2;
best_score = best_score_2;
i++;
@@ -106,6 +110,7 @@ void CreateBackwardReferences(size_t num_bytes,
Command cmd;
cmd.insert_length_ = insert_length;
cmd.copy_length_ = best_len;
cmd.copy_length_code_ = best_len_code;
cmd.copy_distance_ = best_dist;
commands->push_back(cmd);
hasher->set_last_distance(best_dist);
brotli/enc/command.h
+2 -1
View File
@@ -24,13 +24,14 @@ namespace brotli {
// Command holds a sequence of literals and a backward reference copy.
class Command {
public:
Command() : insert_length_(0), copy_length_(0),
Command() : insert_length_(0), copy_length_(0), copy_length_code_(0),
copy_distance_(0), distance_code_(0),
distance_prefix_(0), command_prefix_(0),
distance_extra_bits_(0), distance_extra_bits_value_(0) {}
uint32_t insert_length_;
uint32_t copy_length_;
uint32_t copy_length_code_;
uint32_t copy_distance_;
// Values <= 16 are short codes, values > 16 are distances shifted by 16.
uint32_t distance_code_;
brotli/enc/encode.cc
+33 -28
View File
@@ -34,6 +34,18 @@
namespace brotli {
static const int kWindowBits = 22;
// To make decoding faster, we allow the decoder to write 16 bytes ahead in
// its ringbuffer, therefore the encoder has to decrease max distance by this
// amount.
static const int kDecoderRingBufferWriteAheadSlack = 16;
static const int kMaxBackwardDistance =
(1 << kWindowBits) - kDecoderRingBufferWriteAheadSlack;
static const int kMetaBlockSizeBits = 21;
static const int kRingBufferBits = 23;
static const int kRingBufferMask = (1 << kRingBufferBits) - 1;
template<int kSize>
double Entropy(const std::vector<Histogram<kSize> >& histograms) {
double retval = 0;
@@ -264,7 +276,7 @@ void EncodeCommand(const Command& cmd,
uint64_t insert_extra_bits_val =
cmd.insert_length_ - InsertLengthOffset(code);
int copy_extra_bits = CopyLengthExtraBits(code);
uint64_t copy_extra_bits_val = cmd.copy_length_ - CopyLengthOffset(code);
uint64_t copy_extra_bits_val = cmd.copy_length_code_ - CopyLengthOffset(code);
if (insert_extra_bits > 0) {
WriteBits(insert_extra_bits, insert_extra_bits_val, storage_ix, storage);
}
@@ -325,8 +337,8 @@ void ComputeCommandPrefixes(std::vector<Command>* cmds,
for (int i = 0; i < cmds->size(); ++i) {
Command* cmd = &(*cmds)[i];
cmd->command_prefix_ = CommandPrefix(cmd->insert_length_,
cmd->copy_length_);
if (cmd->copy_length_ > 0) {
cmd->copy_length_code_);
if (cmd->copy_length_code_ > 0) {
PrefixEncodeCopyDistance(cmd->distance_code_,
num_direct_distance_codes,
distance_postfix_bits,
@@ -454,7 +466,7 @@ void EncodeContextMap(const std::vector<int>& context_map,
int* storage_ix, uint8_t* storage) {
WriteBits(8, num_clusters - 1, storage_ix, storage);
if (num_clusters == 1 || num_clusters == context_map.size()) {
if (num_clusters == 1) {
return;
}
@@ -737,10 +749,10 @@ void StoreMetaBlock(const MetaBlock& mb,
}
if (*pos < end_pos && cmd.distance_prefix_ != 0xffff) {
MoveAndEncode(distance_split_code, &distance_it, storage_ix, storage);
int histogram_index = distance_it.type_;
int context = (distance_it.type_ << 2) +
((cmd.copy_length_ > 4) ? 3 : cmd.copy_length_ - 2);
histogram_index = mb.distance_context_map[context];
((cmd.copy_length_code_ > 4) ? 3 : cmd.copy_length_code_ - 2);
int histogram_index = mb.distance_context_map[context];
size_t max_distance = std::min(*pos, (size_t)kMaxBackwardDistance);
EncodeCopyDistance(cmd, distance_codes[histogram_index],
storage_ix, storage);
}
@@ -748,32 +760,21 @@ void StoreMetaBlock(const MetaBlock& mb,
}
}
static const int kWindowBits = 22;
// To make decoding faster, we allow the decoder to write 16 bytes ahead in
// its ringbuffer, therefore the encoder has to decrease max distance by this
// amount.
static const int kDecoderRingBufferWriteAheadSlack = 16;
static const int kMaxBackwardDistance =
(1 << kWindowBits) - kDecoderRingBufferWriteAheadSlack;
static const int kMetaBlockSizeBits = 21;
static const int kRingBufferBits = 23;
static const int kRingBufferMask = (1 << kRingBufferBits) - 1;
BrotliCompressor::BrotliCompressor()
: hasher_(new Hasher),
: window_bits_(kWindowBits),
hasher_(new Hasher),
dist_ringbuffer_idx_(0),
input_pos_(0),
ringbuffer_(kRingBufferBits, kMetaBlockSizeBits),
literal_cost_(1 << kRingBufferBits),
storage_ix_(0),
storage_(new uint8_t[2 << kMetaBlockSizeBits]) {
dist_ringbuffer_[0] = 4;
dist_ringbuffer_[1] = 11;
dist_ringbuffer_[2] = 15;
dist_ringbuffer_[3] = 16;
storage_[0] = 0;
}
dist_ringbuffer_[0] = 4;
dist_ringbuffer_[1] = 11;
dist_ringbuffer_[2] = 15;
dist_ringbuffer_[3] = 16;
storage_[0] = 0;
}
BrotliCompressor::~BrotliCompressor() {
delete hasher_;
@@ -784,8 +785,12 @@ void BrotliCompressor::WriteStreamHeader() {
// Don't encode input size.
WriteBits(3, 0, &storage_ix_, storage_);
// Encode window size.
WriteBits(1, 1, &storage_ix_, storage_);
WriteBits(3, kWindowBits - 17, &storage_ix_, storage_);
if (window_bits_ == 16) {
WriteBits(1, 0, &storage_ix_, storage_);
} else {
WriteBits(1, 1, &storage_ix_, storage_);
WriteBits(3, window_bits_ - 17, &storage_ix_, storage_);
}
}
void BrotliCompressor::WriteMetaBlock(const size_t input_size,
brotli/enc/encode.h
+1
View File
@@ -49,6 +49,7 @@ class BrotliCompressor {
private:
int window_bits_;
Hasher* hasher_;
int dist_ringbuffer_[4];
size_t dist_ringbuffer_idx_;
brotli/enc/hash.h
+42 -34
View File
@@ -147,6 +147,7 @@ class HashLongestMatch {
uint32_t max_length,
const uint32_t max_backward,
size_t * __restrict best_len_out,
size_t * __restrict best_len_code_out,
size_t * __restrict best_distance_out,
double * __restrict best_score_out) {
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@@ -227,6 +228,7 @@ class HashLongestMatch {
best_len = len;
best_ix = backward;
*best_len_out = best_len;
*best_len_code_out = best_len;
*best_distance_out = best_ix;
*best_score_out = best_score;
match_found = true;
@@ -234,7 +236,7 @@ class HashLongestMatch {
}
}
const uint32_t key = Hash3Bytes(&data[cur_ix_masked], kBucketBits);
const uint32_t * __restrict const bucket = &buckets_[key][0];
const int * __restrict const bucket = &buckets_[key][0];
const int down = (num_[key] > kBlockSize) ? (num_[key] - kBlockSize) : 0;
int stop = int(cur_ix) - 64;
if (stop < 0) { stop = 0; }
@@ -259,44 +261,50 @@ class HashLongestMatch {
best_len = len;
best_ix = backward;
*best_len_out = best_len;
*best_len_code_out = best_len;
*best_distance_out = best_ix;
match_found = true;
}
}
for (int i = num_[key] - 1; i >= down; --i) {
size_t prev_ix = bucket[i & kBlockMask];
const size_t backward = cur_ix - prev_ix;
if (PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
int prev_ix = bucket[i & kBlockMask];
if (prev_ix < 0) {
continue;
}
const size_t len =
FindMatchLengthWithLimit(&data[prev_ix], &data[cur_ix_masked],
max_length);
if (len >= 3) {
// Comparing for >= 3 does not change the semantics, but just saves for
// a few unnecessary binary logarithms in backward reference score,
// since we are not interested in such short matches.
const double score = BackwardReferenceScore(average_cost_,
start_cost4,
start_cost3,
start_cost2,
len, backward,
last_distance1_,
last_distance2_,
last_distance3_,
last_distance4_);
if (best_score < score) {
best_score = score;
best_len = len;
best_ix = backward;
*best_len_out = best_len;
*best_distance_out = best_ix;
*best_score_out = best_score;
match_found = true;
} else {
const size_t backward = cur_ix - prev_ix;
if (PREDICT_FALSE(backward > max_backward)) {
break;
}
prev_ix &= ring_buffer_mask;
if (data[cur_ix_masked + best_len] != data[prev_ix + best_len]) {
continue;
}
const size_t len =
FindMatchLengthWithLimit(&data[prev_ix], &data[cur_ix_masked],
max_length);
if (len >= 3) {
// Comparing for >= 3 does not change the semantics, but just saves
// for a few unnecessary binary logarithms in backward reference
// score, since we are not interested in such short matches.
const double score = BackwardReferenceScore(average_cost_,
start_cost4,
start_cost3,
start_cost2,
len, backward,
last_distance1_,
last_distance2_,
last_distance3_,
last_distance4_);
if (best_score < score) {
best_score = score;
best_len = len;
best_ix = backward;
*best_len_out = best_len;
*best_len_code_out = best_len;
*best_distance_out = best_ix;
*best_score_out = best_score;
match_found = true;
}
}
}
}
@@ -333,7 +341,7 @@ class HashLongestMatch {
uint16_t num_[kBucketSize];
// Buckets containing kBlockSize of backward references.
uint32_t buckets_[kBucketSize][kBlockSize];
int buckets_[kBucketSize][kBlockSize];
int last_distance1_;
int last_distance2_;
brotli/enc/histogram.cc
+1 -1
View File
@@ -59,7 +59,7 @@ void BuildHistograms(
if (cmd.copy_length_ > 0 && cmd.distance_prefix_ != 0xffff) {
dist_it.Next();
int context = (dist_it.type_ << kDistanceContextBits) +
((cmd.copy_length_ > 4) ? 3 : cmd.copy_length_ - 2);
((cmd.copy_length_code_ > 4) ? 3 : cmd.copy_length_code_ - 2);
(*copy_dist_histograms)[context].Add(cmd.distance_prefix_);
}
}