Add a simplified runner which outputs a CSV report

This commit is contained in:
David Kuettel
2013-04-11 17:40:54 -07:00
parent 19411b7e95
commit 42726b6c1f
7 changed files with 391 additions and 172 deletions
+1 -1
View File
@@ -23,7 +23,7 @@
<zipfileset src="lib/woffconverter.jar" />
<zipfileset src="lib/lzma.jar" />
<manifest>
<attribute name="Main-Class" value="com.google.typography.font.compression.CompressionRunner" />
<attribute name="Main-Class" value="com.google.typography.font.compression.SimpleRunner" />
</manifest>
</jar>
</target>
@@ -4,40 +4,26 @@
package com.google.typography.font.compression;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import com.google.typography.font.sfntly.Font;
import com.google.typography.font.sfntly.FontFactory;
import com.google.typography.font.sfntly.Tag;
import com.google.typography.font.sfntly.data.ReadableFontData;
import com.google.typography.font.tools.conversion.eot.EOTWriter;
import com.google.typography.font.tools.conversion.eot.HdmxEncoder;
import com.google.typography.font.tools.conversion.woff.WoffWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
/**
* A command-line tool for running different experimental compression code over
* a corpus of fonts, and gathering statistics, particularly compression
* efficiency.
*
* This is not intended to be production code, and for certain codecs it will
* shell out to helper binaries, which is fine for the purposes of gathering
* statistics, but obviously not much else.
* This is not intended to be production code.
*
* @author Raph Levien
*/
public class CompressionRunner {
// private static final boolean DEBUG = false;
public static void main(String[] args) throws IOException {
boolean generateOutput = false;
@@ -67,11 +53,11 @@ public class CompressionRunner {
if (descs.isEmpty()) {
descs.add("glyf/cbbox,triplet,code,reslice:woff2/lzma");
}
runTest(filenames, baseline, descs, generateOutput);
run(filenames, baseline, descs, generateOutput);
}
private static void runTest(List<String> filenames, String baseline, List<String> descs,
boolean generateOutput) throws IOException {
private static void run(List<String> filenames, String baseline, List<String> descs,
boolean generateOutput) throws IOException {
PrintWriter o = new PrintWriter(System.out);
List<StatsCollector> stats = Lists.newArrayList();
for (int i = 0; i < descs.size(); i++) {
@@ -82,10 +68,10 @@ public class CompressionRunner {
for (String filename : filenames) {
byte[] bytes = Files.toByteArray(new File(filename));
Font font = fontFactory.loadFonts(bytes)[0];
byte[] baselineResult = runExperiment(font, baseline);
byte[] baselineResult = Experiment.run(font, baseline);
o.printf("<!-- %s: baseline %d bytes", new File(filename).getName(), baselineResult.length);
for (int i = 0; i < descs.size(); i++) {
byte[] expResult = runExperiment(font, descs.get(i));
byte[] expResult = Experiment.run(font, descs.get(i));
if (generateOutput) {
String newFilename = filename;
if (newFilename.endsWith(".ttf")) {
@@ -114,155 +100,4 @@ public class CompressionRunner {
stats.get(0).chartFooter(o);
o.close();
}
private static Font.Builder stripTags(Font srcFont, Set<Integer> removeTags) {
FontFactory fontFactory = FontFactory.getInstance();
Font.Builder fontBuilder = fontFactory.newFontBuilder();
for (Integer tag : srcFont.tableMap().keySet()) {
if (!removeTags.contains(tag)) {
fontBuilder.newTableBuilder(tag, srcFont.getTable(tag).readFontData());
}
}
return fontBuilder;
}
private static Font.Builder preprocessMtxGlyf(Font srcFont, String options) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.<Integer>of());
GlyfEncoder glyfEncoder = new GlyfEncoder(options);
glyfEncoder.encode(srcFont);
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '1'}),
glyfEncoder.getGlyfBytes());
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'l', 'o', 'c', 'z'}),
glyfEncoder.getLocaBytes());
if (!Arrays.asList(options.split(",")).contains("reslice")) {
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '2'}),
glyfEncoder.getCodeBytes());
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '3'}),
glyfEncoder.getPushBytes());
}
return fontBuilder;
}
private static Font.Builder preprocessHmtx(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hmtx));
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'h', 'm', 't', 'z'}),
toBytes(AdvWidth.encode(srcFont)));
return fontBuilder;
}
private static Font.Builder preprocessHdmx(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hdmx));
if (srcFont.hasTable(Tag.hdmx)) {
addTableBytes(fontBuilder, Tag.hdmx, toBytes(new HdmxEncoder().encode(srcFont)));
}
return fontBuilder;
}
private static Font.Builder preprocessCmap(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.cmap));
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'c', 'm', 'a', 'z'}),
CmapEncoder.encode(srcFont));
return fontBuilder;
}
private static Font.Builder preprocessKern(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.kern));
if (srcFont.hasTable(Tag.kern)) {
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'k', 'e', 'r', 'z'}),
toBytes(KernEncoder.encode(srcFont)));
}
return fontBuilder;
}
private static void addTableBytes(Font.Builder fontBuilder, int tag, byte[] contents) {
fontBuilder.newTableBuilder(tag, ReadableFontData.createReadableFontData(contents));
}
private static byte[] fontToBytes(FontFactory fontFactory, Font font) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
fontFactory.serializeFont(font, baos);
return baos.toByteArray();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static byte[] toBytes(ReadableFontData rfd) {
byte[] result = new byte[rfd.length()];
rfd.readBytes(0, result, 0, rfd.length());
return result;
}
/**
* Does one experimental compression on a font, using the string to guide what
* gets done.
*
* @param srcFont Source font
* @param desc experiment description string; the exact format is probably
* still evolving
* @return serialization of compressed font
* @throws IOException
*/
private static byte[] runExperiment(Font srcFont, String desc) throws IOException {
Font font = srcFont;
FontFactory fontFactory = FontFactory.getInstance();
String[] pieces = desc.split(":");
boolean keepDsig = false;
for (int i = 0; i < pieces.length - 1; i++) {
String[] piece = pieces[i].split("/");
String cmd = piece[0];
if (cmd.equals("glyf")) {
font = preprocessMtxGlyf(font, piece.length > 1 ? piece[1] : "").build();
} else if (cmd.equals("hmtx")) {
font = preprocessHmtx(font).build();
} else if (cmd.equals("hdmx")) {
font = preprocessHdmx(font).build();
} else if (cmd.equals("cmap")) {
font = preprocessCmap(font).build();
} else if (cmd.equals("kern")) {
font = preprocessKern(font).build();
} else if (cmd.equals("keepdsig")) {
keepDsig = true;
} else if (cmd.equals("strip")) {
Set<Integer> removeTags = Sets.newTreeSet();
for (String tag : piece[1].split(",")) {
removeTags.add(Tag.intValue(tag.getBytes()));
}
font = stripTags(font, removeTags).build();
}
}
if (!keepDsig) {
font = stripTags(font, ImmutableSet.of(Tag.DSIG)).build();
}
String last = pieces[pieces.length - 1];
String[] lastPieces = last.split("/");
String lastBase = lastPieces[0];
String lastArgs = lastPieces.length > 1 ? lastPieces[1] : "";
if (!lastBase.equals("woff2")) {
Set<Integer> tagsToStrip = Sets.newHashSet();
for (Entry<Integer, Integer> mapping : Woff2Writer.getTransformMap().entrySet()) {
if (font.hasTable(mapping.getValue())) {
tagsToStrip.add(mapping.getKey());
}
}
font = stripTags(font, tagsToStrip).build();
}
byte[] result = null;
if (lastBase.equals("gzip")) {
result = GzipUtil.deflate(fontToBytes(fontFactory, font));
} else if (lastBase.equals("lzma")) {
result = CompressLzma.compress(fontToBytes(fontFactory, font));
} else if (lastBase.equals("woff")) {
result = toBytes(new WoffWriter().convert(font));
} else if (lastBase.equals("woff2")) {
result = toBytes(new Woff2Writer(lastArgs).convert(font));
} else if (lastBase.equals("eot")) {
result = toBytes(new EOTWriter(true).convert(font));
} else if (lastBase.equals("uncomp")) {
result = fontToBytes(fontFactory, font);
}
return result;
}
}
@@ -0,0 +1,99 @@
package com.google.typography.font.compression;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* Compression stats, both aggregate and per font.
*
* @author David Kuettel
*/
public class CompressionStats {
public enum Size { ORIGINAL, GZIP, WOFF2 }
private final List<Stats> values = Lists.newArrayList();
public void add(Stats stat) {
values.add(stat);
}
public List<Stats> values() {
return values;
}
public double mean(Size size) {
double sum = 0;
for (Long value : values(size)) {
sum += value;
}
return sum / values.size();
}
public double median(Size size) {
List<Long> list = values(size);
Collections.sort(list);
int length = list.size();
if (length % 2 == 1) {
return list.get((length - 1) / 2);
} else {
return 0.5 * (list.get(length / 2 - 1) + list.get(length / 2));
}
}
private List<Long> values(Size size) {
List<Long> list = Lists.newArrayList();
for (Stats stats : values) {
list.add(stats.getSize(size));
}
return list;
}
public static class Stats {
private final String filename;
private final Map<Size, Long> sizes;
private Stats(String filename, Map<Size, Long> sizes) {
this.filename = filename;
this.sizes = sizes;
}
public String getFilename() {
return filename;
}
public long getSize(Size size) {
return sizes.get(size);
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String filename;
private Map<Size, Long> sizes = Maps.newHashMap();
public Builder setFilename(String filename) {
this.filename = filename;
return this;
}
public Builder setSize(Size key, long value) {
this.sizes.put(key, value);
return this;
}
public Stats build() {
return new Stats(filename, ImmutableMap.copyOf(sizes));
}
}
}
}
@@ -0,0 +1,32 @@
package com.google.typography.font.compression;
import com.google.common.io.Closeables;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
/**
* Generates a CSV report containing the compression stats.
*
* @author David Kuettel
*/
public class CsvReport {
public static void create(CompressionStats stats, String filename) throws IOException {
PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter(filename)));
try {
writer.printf("Font, Original, GZIP, WOFF 2.0\n");
for (CompressionStats.Stats stat : stats.values()) {
writer.printf("%s, %d, %d, %d\n",
stat.getFilename(),
stat.getSize(CompressionStats.Size.ORIGINAL),
stat.getSize(CompressionStats.Size.GZIP),
stat.getSize(CompressionStats.Size.WOFF2));
}
} finally {
Closeables.closeQuietly(writer);
}
}
}
@@ -0,0 +1,93 @@
package com.google.typography.font.compression;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.typography.font.sfntly.Font;
import com.google.typography.font.sfntly.FontFactory;
import com.google.typography.font.sfntly.Tag;
import com.google.typography.font.tools.conversion.eot.EOTWriter;
import com.google.typography.font.tools.conversion.woff.WoffWriter;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
/**
* @author Raph Levien
*/
public class Experiment {
/**
* Does one experimental compression on a font, using the string to guide what
* gets done.
*
* @param srcFont Source font
* @param desc experiment description string; the exact format is probably
* still evolving
* @return serialization of compressed font
* @throws IOException
*/
public static byte[] run(Font srcFont, String desc) throws IOException {
Font font = srcFont;
FontFactory fontFactory = FontFactory.getInstance();
String[] pieces = desc.split(":");
boolean keepDsig = false;
for (int i = 0; i < pieces.length - 1; i++) {
String[] piece = pieces[i].split("/");
String cmd = piece[0];
if (cmd.equals("glyf")) {
font = FontUtil.preprocessMtxGlyf(font, piece.length > 1 ? piece[1] : "").build();
} else if (cmd.equals("hmtx")) {
font = FontUtil.preprocessHmtx(font).build();
} else if (cmd.equals("hdmx")) {
font = FontUtil.preprocessHdmx(font).build();
} else if (cmd.equals("cmap")) {
font = FontUtil.preprocessCmap(font).build();
} else if (cmd.equals("kern")) {
font = FontUtil.preprocessKern(font).build();
} else if (cmd.equals("keepdsig")) {
keepDsig = true;
} else if (cmd.equals("strip")) {
Set<Integer> removeTags = Sets.newTreeSet();
for (String tag : piece[1].split(",")) {
removeTags.add(Tag.intValue(tag.getBytes()));
}
font = FontUtil.stripTags(font, removeTags).build();
}
}
if (!keepDsig) {
font = FontUtil.stripTags(font, ImmutableSet.of(Tag.DSIG)).build();
}
String last = pieces[pieces.length - 1];
String[] lastPieces = last.split("/");
String lastBase = lastPieces[0];
String lastArgs = lastPieces.length > 1 ? lastPieces[1] : "";
if (!lastBase.equals("woff2")) {
Set<Integer> tagsToStrip = Sets.newHashSet();
for (Map.Entry<Integer, Integer> mapping : Woff2Writer.getTransformMap().entrySet()) {
if (font.hasTable(mapping.getValue())) {
tagsToStrip.add(mapping.getKey());
}
}
font = FontUtil.stripTags(font, tagsToStrip).build();
}
byte[] result = null;
if (lastBase.equals("gzip")) {
result = GzipUtil.deflate(FontUtil.toBytes(fontFactory, font));
} else if (lastBase.equals("lzma")) {
result = CompressLzma.compress(FontUtil.toBytes(fontFactory, font));
} else if (lastBase.equals("woff")) {
result = FontUtil.toBytes(new WoffWriter().convert(font));
} else if (lastBase.equals("woff2")) {
result = FontUtil.toBytes(new Woff2Writer(lastArgs).convert(font));
} else if (lastBase.equals("eot")) {
result = FontUtil.toBytes(new EOTWriter(true).convert(font));
} else if (lastBase.equals("uncomp")) {
result = FontUtil.toBytes(fontFactory, font);
}
return result;
}
}
@@ -0,0 +1,100 @@
package com.google.typography.font.compression;
import com.google.common.collect.ImmutableSet;
import com.google.typography.font.sfntly.Font;
import com.google.typography.font.sfntly.FontFactory;
import com.google.typography.font.sfntly.Tag;
import com.google.typography.font.sfntly.data.ReadableFontData;
import com.google.typography.font.tools.conversion.eot.HdmxEncoder;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.Set;
/**
* Font utility methods
*
* @author Raph Levien
*/
public class FontUtil {
public static Font.Builder stripTags(Font srcFont, Set<Integer> removeTags) {
FontFactory fontFactory = FontFactory.getInstance();
Font.Builder fontBuilder = fontFactory.newFontBuilder();
for (Integer tag : srcFont.tableMap().keySet()) {
if (!removeTags.contains(tag)) {
fontBuilder.newTableBuilder(tag, srcFont.getTable(tag).readFontData());
}
}
return fontBuilder;
}
public static Font.Builder preprocessMtxGlyf(Font srcFont, String options) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.<Integer>of());
GlyfEncoder glyfEncoder = new GlyfEncoder(options);
glyfEncoder.encode(srcFont);
addTableBytes(fontBuilder, Tag.intValue(new byte[]{'g', 'l', 'z', '1'}),
glyfEncoder.getGlyfBytes());
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'l', 'o', 'c', 'z'}),
glyfEncoder.getLocaBytes());
if (!Arrays.asList(options.split(",")).contains("reslice")) {
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '2'}),
glyfEncoder.getCodeBytes());
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '3'}),
glyfEncoder.getPushBytes());
}
return fontBuilder;
}
public static Font.Builder preprocessHmtx(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hmtx));
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'h', 'm', 't', 'z'}),
toBytes(AdvWidth.encode(srcFont)));
return fontBuilder;
}
public static Font.Builder preprocessHdmx(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hdmx));
if (srcFont.hasTable(Tag.hdmx)) {
addTableBytes(fontBuilder, Tag.hdmx, toBytes(new HdmxEncoder().encode(srcFont)));
}
return fontBuilder;
}
public static Font.Builder preprocessCmap(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.cmap));
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'c', 'm', 'a', 'z'}),
CmapEncoder.encode(srcFont));
return fontBuilder;
}
public static Font.Builder preprocessKern(Font srcFont) {
Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.kern));
if (srcFont.hasTable(Tag.kern)) {
addTableBytes(fontBuilder, Tag.intValue(new byte[] {'k', 'e', 'r', 'z'}),
toBytes(KernEncoder.encode(srcFont)));
}
return fontBuilder;
}
public static void addTableBytes(Font.Builder fontBuilder, int tag, byte[] contents) {
fontBuilder.newTableBuilder(tag, ReadableFontData.createReadableFontData(contents));
}
public static byte[] toBytes(FontFactory fontFactory, Font font) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
fontFactory.serializeFont(font, baos);
return baos.toByteArray();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static byte[] toBytes(ReadableFontData rfd) {
byte[] result = new byte[rfd.length()];
rfd.readBytes(0, result, 0, rfd.length());
return result;
}
}
@@ -0,0 +1,60 @@
package com.google.typography.font.compression;
import com.google.common.io.Files;
import com.google.typography.font.sfntly.Font;
import com.google.typography.font.sfntly.FontFactory;
import java.io.File;
import java.io.IOException;
/**
* Simple WOFF 2.0 compression report runner.
*
* @author David Kuettel
*/
public class SimpleRunner {
private static final FontFactory FONT_FACTORY = FontFactory.getInstance();
private static final String GZIP = "gzip";
private static final String WOFF2 = "glyf/cbbox,triplet,code,reslice:woff2/lzma";
private static final String REPORT = "report.csv";
public static void main(String[] args) throws IOException {
if (args.length == 0) {
usage();
}
CompressionStats stats = new CompressionStats();
System.out.printf("Analyzing (%d) fonts\n", args.length);
run(stats, args);
System.out.printf("Creating report: %s\n", REPORT);
CsvReport.create(stats, REPORT);
}
private static void run(CompressionStats stats, String[] filenames) throws IOException {
for (String filename : filenames) {
File file = new File(filename);
byte[] bytes = Files.toByteArray(file);
Font font = FONT_FACTORY.loadFonts(bytes)[0];
byte[] gzip = Experiment.run(font, GZIP);
byte[] woff2 = Experiment.run(font, WOFF2);
stats.add(
CompressionStats.Stats.builder()
.setFilename(file.getName())
.setSize(CompressionStats.Size.ORIGINAL, bytes.length)
.setSize(CompressionStats.Size.GZIP, gzip.length)
.setSize(CompressionStats.Size.WOFF2, woff2.length)
.build());
}
}
private static void usage() {
System.err.println("Usage: SimpleRunner font...");
System.exit(-1);
}
}