Add a simplified runner which outputs a CSV report

build.xml

@@ -23,7 +23,7 @@
       <zipfileset src="lib/woffconverter.jar" />
       <zipfileset src="lib/lzma.jar" />
       <manifest>
-        <attribute name="Main-Class" value="com.google.typography.font.compression.CompressionRunner" />
+        <attribute name="Main-Class" value="com.google.typography.font.compression.SimpleRunner" />
       </manifest>
     </jar>
   </target>

src/com/google/typography/font/compression/CompressionRunner.java

@@ -4,40 +4,26 @@
 
 package com.google.typography.font.compression;
 
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 import com.google.common.io.Files;
 import com.google.typography.font.sfntly.Font;
 import com.google.typography.font.sfntly.FontFactory;
-import com.google.typography.font.sfntly.Tag;
-import com.google.typography.font.sfntly.data.ReadableFontData;
-import com.google.typography.font.tools.conversion.eot.EOTWriter;
-import com.google.typography.font.tools.conversion.eot.HdmxEncoder;
-import com.google.typography.font.tools.conversion.woff.WoffWriter;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
 
 /**
  * A command-line tool for running different experimental compression code over
  * a corpus of fonts, and gathering statistics, particularly compression
  * efficiency.
  *
- * This is not intended to be production code, and for certain codecs it will
+ * This is not intended to be production code.
- * shell out to helper binaries, which is fine for the purposes of gathering
- * statistics, but obviously not much else.
  *
  * @author Raph Levien
  */
 public class CompressionRunner {
-  // private static final boolean DEBUG = false;
 
   public static void main(String[] args) throws IOException {
     boolean generateOutput = false;
@@ -67,10 +53,10 @@ public class CompressionRunner {
     if (descs.isEmpty()) {
       descs.add("glyf/cbbox,triplet,code,reslice:woff2/lzma");
     }
-    runTest(filenames, baseline, descs, generateOutput);
+    run(filenames, baseline, descs, generateOutput);
   }
 
-  private static void runTest(List<String> filenames, String baseline, List<String> descs,
+  private static void run(List<String> filenames, String baseline, List<String> descs,
                           boolean generateOutput) throws IOException {
     PrintWriter o = new PrintWriter(System.out);
     List<StatsCollector> stats = Lists.newArrayList();
@@ -82,10 +68,10 @@ public class CompressionRunner {
     for (String filename : filenames) {
       byte[] bytes = Files.toByteArray(new File(filename));
       Font font = fontFactory.loadFonts(bytes)[0];
-      byte[] baselineResult = runExperiment(font, baseline);
+      byte[] baselineResult = Experiment.run(font, baseline);
       o.printf("<!-- %s: baseline %d bytes", new File(filename).getName(), baselineResult.length);
       for (int i = 0; i < descs.size(); i++) {
-        byte[] expResult = runExperiment(font, descs.get(i));
+        byte[] expResult = Experiment.run(font, descs.get(i));
         if (generateOutput) {
           String newFilename = filename;
           if (newFilename.endsWith(".ttf")) {
@@ -114,155 +100,4 @@ public class CompressionRunner {
     stats.get(0).chartFooter(o);
     o.close();
   }
-
-  private static Font.Builder stripTags(Font srcFont, Set<Integer> removeTags) {
-    FontFactory fontFactory = FontFactory.getInstance();
-    Font.Builder fontBuilder = fontFactory.newFontBuilder();
-
-    for (Integer tag : srcFont.tableMap().keySet()) {
-      if (!removeTags.contains(tag)) {
-        fontBuilder.newTableBuilder(tag, srcFont.getTable(tag).readFontData());
-      }
-    }
-    return fontBuilder;
-  }
-
-  private static Font.Builder preprocessMtxGlyf(Font srcFont, String options) {
-    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.<Integer>of());
-    GlyfEncoder glyfEncoder = new GlyfEncoder(options);
-    glyfEncoder.encode(srcFont);
-    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '1'}),
-        glyfEncoder.getGlyfBytes());
-    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'l', 'o', 'c', 'z'}),
-        glyfEncoder.getLocaBytes());
-    if (!Arrays.asList(options.split(",")).contains("reslice")) {
-      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '2'}),
-          glyfEncoder.getCodeBytes());
-      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '3'}),
-          glyfEncoder.getPushBytes());
-    }
-    return fontBuilder;
-  }
-
-  private static Font.Builder preprocessHmtx(Font srcFont) {
-    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hmtx));
-    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'h', 'm', 't', 'z'}),
-        toBytes(AdvWidth.encode(srcFont)));
-    return fontBuilder;
-  }
-
-  private static Font.Builder preprocessHdmx(Font srcFont) {
-    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hdmx));
-    if (srcFont.hasTable(Tag.hdmx)) {
-      addTableBytes(fontBuilder, Tag.hdmx, toBytes(new HdmxEncoder().encode(srcFont)));
-    }
-    return fontBuilder;
-  }
-
-  private static Font.Builder preprocessCmap(Font srcFont) {
-    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.cmap));
-    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'c', 'm', 'a', 'z'}),
-        CmapEncoder.encode(srcFont));
-    return fontBuilder;
-  }
-
-  private static Font.Builder preprocessKern(Font srcFont) {
-    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.kern));
-    if (srcFont.hasTable(Tag.kern)) {
-      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'k', 'e', 'r', 'z'}),
-          toBytes(KernEncoder.encode(srcFont)));
-    }
-    return fontBuilder;
-  }
-
-  private static void addTableBytes(Font.Builder fontBuilder, int tag, byte[] contents) {
-    fontBuilder.newTableBuilder(tag, ReadableFontData.createReadableFontData(contents));
-  }
-
-  private static byte[] fontToBytes(FontFactory fontFactory, Font font) {
-    try {
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      fontFactory.serializeFont(font, baos);
-      return baos.toByteArray();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private static byte[] toBytes(ReadableFontData rfd) {
-    byte[] result = new byte[rfd.length()];
-    rfd.readBytes(0, result, 0, rfd.length());
-    return result;
-  }
-
-  /**
-   * Does one experimental compression on a font, using the string to guide what
-   * gets done.
-   *
-   * @param srcFont Source font
-   * @param desc experiment description string; the exact format is probably
-   *        still evolving
-   * @return serialization of compressed font
-   * @throws IOException
-   */
-  private static byte[] runExperiment(Font srcFont, String desc) throws IOException {
-    Font font = srcFont;
-    FontFactory fontFactory = FontFactory.getInstance();
-    String[] pieces = desc.split(":");
-    boolean keepDsig = false;
-    for (int i = 0; i < pieces.length - 1; i++) {
-      String[] piece = pieces[i].split("/");
-      String cmd = piece[0];
-      if (cmd.equals("glyf")) {
-        font = preprocessMtxGlyf(font, piece.length > 1 ? piece[1] : "").build();
-      } else if (cmd.equals("hmtx")) {
-        font = preprocessHmtx(font).build();
-      } else if (cmd.equals("hdmx")) {
-        font = preprocessHdmx(font).build();
-      } else if (cmd.equals("cmap")) {
-        font = preprocessCmap(font).build();
-      } else if (cmd.equals("kern")) {
-        font = preprocessKern(font).build();
-      } else if (cmd.equals("keepdsig")) {
-        keepDsig = true;
-      } else if (cmd.equals("strip")) {
-        Set<Integer> removeTags = Sets.newTreeSet();
-        for (String tag : piece[1].split(",")) {
-          removeTags.add(Tag.intValue(tag.getBytes()));
-        }
-        font = stripTags(font, removeTags).build();
-      }
-    }
-    if (!keepDsig) {
-      font = stripTags(font, ImmutableSet.of(Tag.DSIG)).build();
-    }
-    String last = pieces[pieces.length - 1];
-    String[] lastPieces = last.split("/");
-    String lastBase = lastPieces[0];
-    String lastArgs = lastPieces.length > 1 ? lastPieces[1] : "";
-    if (!lastBase.equals("woff2")) {
-      Set<Integer> tagsToStrip = Sets.newHashSet();
-      for (Entry<Integer, Integer> mapping : Woff2Writer.getTransformMap().entrySet()) {
-        if (font.hasTable(mapping.getValue())) {
-          tagsToStrip.add(mapping.getKey());
-        }
-      }
-      font = stripTags(font, tagsToStrip).build();
-    }
-    byte[] result = null;
-    if (lastBase.equals("gzip")) {
-      result = GzipUtil.deflate(fontToBytes(fontFactory, font));
-    } else if (lastBase.equals("lzma")) {
-      result = CompressLzma.compress(fontToBytes(fontFactory, font));
-    } else if (lastBase.equals("woff")) {
-      result = toBytes(new WoffWriter().convert(font));
-    } else if (lastBase.equals("woff2")) {
-      result = toBytes(new Woff2Writer(lastArgs).convert(font));
-    } else if (lastBase.equals("eot")) {
-      result = toBytes(new EOTWriter(true).convert(font));
-    } else if (lastBase.equals("uncomp")) {
-      result = fontToBytes(fontFactory, font);
-    }
-    return result;
-  }
 }

src/com/google/typography/font/compression/CompressionStats.java

@@ -0,0 +1,99 @@
+package com.google.typography.font.compression;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Compression stats, both aggregate and per font.
+ *
+ * @author David Kuettel
+ */
+public class CompressionStats {
+
+  public enum Size { ORIGINAL, GZIP, WOFF2 }
+
+  private final List<Stats> values = Lists.newArrayList();
+
+  public void add(Stats stat) {
+    values.add(stat);
+  }
+
+  public List<Stats> values() {
+    return values;
+  }
+
+  public double mean(Size size) {
+    double sum = 0;
+    for (Long value : values(size)) {
+      sum += value;
+    }
+    return sum / values.size();
+  }
+
+  public double median(Size size) {
+    List<Long> list = values(size);
+    Collections.sort(list);
+    int length = list.size();
+    if (length % 2 == 1) {
+      return list.get((length - 1) / 2);
+    } else {
+      return 0.5 * (list.get(length / 2 - 1) + list.get(length / 2));
+    }
+  }
+
+  private List<Long> values(Size size) {
+    List<Long> list = Lists.newArrayList();
+    for (Stats stats : values) {
+      list.add(stats.getSize(size));
+    }
+    return list;
+  }
+
+  public static class Stats {
+
+    private final String filename;
+    private final Map<Size, Long> sizes;
+
+    private Stats(String filename, Map<Size, Long> sizes) {
+      this.filename = filename;
+      this.sizes = sizes;
+    }
+
+    public String getFilename() {
+      return filename;
+    }
+
+    public long getSize(Size size) {
+      return sizes.get(size);
+    }
+
+    public static Builder builder() {
+      return new Builder();
+    }
+
+    public static class Builder {
+
+      private String filename;
+      private Map<Size, Long> sizes = Maps.newHashMap();
+
+      public Builder setFilename(String filename) {
+        this.filename = filename;
+        return this;
+      }
+
+      public Builder setSize(Size key, long value) {
+        this.sizes.put(key, value);
+        return this;
+      }
+
+      public Stats build() {
+        return new Stats(filename, ImmutableMap.copyOf(sizes));
+      }
+    }
+  }
+}

src/com/google/typography/font/compression/CsvReport.java

@@ -0,0 +1,32 @@
+package com.google.typography.font.compression;
+
+import com.google.common.io.Closeables;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Generates a CSV report containing the compression stats.
+ *
+ * @author David Kuettel
+ */
+public class CsvReport {
+
+  public static void create(CompressionStats stats, String filename) throws IOException {
+    PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter(filename)));
+    try {
+      writer.printf("Font, Original, GZIP, WOFF 2.0\n");
+      for (CompressionStats.Stats stat : stats.values()) {
+        writer.printf("%s, %d, %d, %d\n",
+            stat.getFilename(),
+            stat.getSize(CompressionStats.Size.ORIGINAL),
+            stat.getSize(CompressionStats.Size.GZIP),
+            stat.getSize(CompressionStats.Size.WOFF2));
+      }
+    } finally {
+      Closeables.closeQuietly(writer);
+    }
+  }
+}

src/com/google/typography/font/compression/Experiment.java

@@ -0,0 +1,93 @@
+package com.google.typography.font.compression;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.typography.font.sfntly.Font;
+import com.google.typography.font.sfntly.FontFactory;
+import com.google.typography.font.sfntly.Tag;
+import com.google.typography.font.tools.conversion.eot.EOTWriter;
+import com.google.typography.font.tools.conversion.woff.WoffWriter;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Raph Levien
+ */
+public class Experiment {
+
+  /**
+   * Does one experimental compression on a font, using the string to guide what
+   * gets done.
+   *
+   * @param srcFont Source font
+   * @param desc experiment description string; the exact format is probably
+   *        still evolving
+   * @return serialization of compressed font
+   * @throws IOException
+   */
+  public static byte[] run(Font srcFont, String desc) throws IOException {
+    Font font = srcFont;
+    FontFactory fontFactory = FontFactory.getInstance();
+    String[] pieces = desc.split(":");
+    boolean keepDsig = false;
+
+    for (int i = 0; i < pieces.length - 1; i++) {
+      String[] piece = pieces[i].split("/");
+      String cmd = piece[0];
+      if (cmd.equals("glyf")) {
+        font = FontUtil.preprocessMtxGlyf(font, piece.length > 1 ? piece[1] : "").build();
+      } else if (cmd.equals("hmtx")) {
+        font = FontUtil.preprocessHmtx(font).build();
+      } else if (cmd.equals("hdmx")) {
+        font = FontUtil.preprocessHdmx(font).build();
+      } else if (cmd.equals("cmap")) {
+        font = FontUtil.preprocessCmap(font).build();
+      } else if (cmd.equals("kern")) {
+        font = FontUtil.preprocessKern(font).build();
+      } else if (cmd.equals("keepdsig")) {
+        keepDsig = true;
+      } else if (cmd.equals("strip")) {
+        Set<Integer> removeTags = Sets.newTreeSet();
+        for (String tag : piece[1].split(",")) {
+          removeTags.add(Tag.intValue(tag.getBytes()));
+        }
+        font = FontUtil.stripTags(font, removeTags).build();
+      }
+    }
+    if (!keepDsig) {
+      font = FontUtil.stripTags(font, ImmutableSet.of(Tag.DSIG)).build();
+    }
+
+    String last = pieces[pieces.length - 1];
+    String[] lastPieces = last.split("/");
+    String lastBase = lastPieces[0];
+    String lastArgs = lastPieces.length > 1 ? lastPieces[1] : "";
+    if (!lastBase.equals("woff2")) {
+      Set<Integer> tagsToStrip = Sets.newHashSet();
+      for (Map.Entry<Integer, Integer> mapping : Woff2Writer.getTransformMap().entrySet()) {
+        if (font.hasTable(mapping.getValue())) {
+          tagsToStrip.add(mapping.getKey());
+        }
+      }
+      font = FontUtil.stripTags(font, tagsToStrip).build();
+    }
+
+    byte[] result = null;
+    if (lastBase.equals("gzip")) {
+      result = GzipUtil.deflate(FontUtil.toBytes(fontFactory, font));
+    } else if (lastBase.equals("lzma")) {
+      result = CompressLzma.compress(FontUtil.toBytes(fontFactory, font));
+    } else if (lastBase.equals("woff")) {
+      result = FontUtil.toBytes(new WoffWriter().convert(font));
+    } else if (lastBase.equals("woff2")) {
+      result = FontUtil.toBytes(new Woff2Writer(lastArgs).convert(font));
+    } else if (lastBase.equals("eot")) {
+      result = FontUtil.toBytes(new EOTWriter(true).convert(font));
+    } else if (lastBase.equals("uncomp")) {
+      result = FontUtil.toBytes(fontFactory, font);
+    }
+    return result;
+  }
+}

src/com/google/typography/font/compression/FontUtil.java

@@ -0,0 +1,100 @@
+package com.google.typography.font.compression;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.typography.font.sfntly.Font;
+import com.google.typography.font.sfntly.FontFactory;
+import com.google.typography.font.sfntly.Tag;
+import com.google.typography.font.sfntly.data.ReadableFontData;
+import com.google.typography.font.tools.conversion.eot.HdmxEncoder;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+import java.util.Set;
+
+/**
+ * Font utility methods
+ *
+ * @author Raph Levien
+ */
+public class FontUtil {
+
+  public static Font.Builder stripTags(Font srcFont, Set<Integer> removeTags) {
+    FontFactory fontFactory = FontFactory.getInstance();
+    Font.Builder fontBuilder = fontFactory.newFontBuilder();
+
+    for (Integer tag : srcFont.tableMap().keySet()) {
+      if (!removeTags.contains(tag)) {
+        fontBuilder.newTableBuilder(tag, srcFont.getTable(tag).readFontData());
+      }
+    }
+    return fontBuilder;
+  }
+
+  public static Font.Builder preprocessMtxGlyf(Font srcFont, String options) {
+    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.<Integer>of());
+    GlyfEncoder glyfEncoder = new GlyfEncoder(options);
+    glyfEncoder.encode(srcFont);
+    addTableBytes(fontBuilder, Tag.intValue(new byte[]{'g', 'l', 'z', '1'}),
+        glyfEncoder.getGlyfBytes());
+    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'l', 'o', 'c', 'z'}),
+        glyfEncoder.getLocaBytes());
+    if (!Arrays.asList(options.split(",")).contains("reslice")) {
+      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '2'}),
+          glyfEncoder.getCodeBytes());
+      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'g', 'l', 'z', '3'}),
+          glyfEncoder.getPushBytes());
+    }
+    return fontBuilder;
+  }
+
+  public static Font.Builder preprocessHmtx(Font srcFont) {
+    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hmtx));
+    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'h', 'm', 't', 'z'}),
+        toBytes(AdvWidth.encode(srcFont)));
+    return fontBuilder;
+  }
+
+  public static Font.Builder preprocessHdmx(Font srcFont) {
+    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.hdmx));
+    if (srcFont.hasTable(Tag.hdmx)) {
+      addTableBytes(fontBuilder, Tag.hdmx, toBytes(new HdmxEncoder().encode(srcFont)));
+    }
+    return fontBuilder;
+  }
+
+  public static Font.Builder preprocessCmap(Font srcFont) {
+    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.cmap));
+    addTableBytes(fontBuilder, Tag.intValue(new byte[] {'c', 'm', 'a', 'z'}),
+        CmapEncoder.encode(srcFont));
+    return fontBuilder;
+  }
+
+  public static Font.Builder preprocessKern(Font srcFont) {
+    Font.Builder fontBuilder = stripTags(srcFont, ImmutableSet.of(Tag.kern));
+    if (srcFont.hasTable(Tag.kern)) {
+      addTableBytes(fontBuilder, Tag.intValue(new byte[] {'k', 'e', 'r', 'z'}),
+          toBytes(KernEncoder.encode(srcFont)));
+    }
+    return fontBuilder;
+  }
+
+  public static void addTableBytes(Font.Builder fontBuilder, int tag, byte[] contents) {
+    fontBuilder.newTableBuilder(tag, ReadableFontData.createReadableFontData(contents));
+  }
+
+  public static byte[] toBytes(FontFactory fontFactory, Font font) {
+    try {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      fontFactory.serializeFont(font, baos);
+      return baos.toByteArray();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static byte[] toBytes(ReadableFontData rfd) {
+    byte[] result = new byte[rfd.length()];
+    rfd.readBytes(0, result, 0, rfd.length());
+    return result;
+  }
+}

src/com/google/typography/font/compression/SimpleRunner.java

@@ -0,0 +1,60 @@
+package com.google.typography.font.compression;
+
+import com.google.common.io.Files;
+import com.google.typography.font.sfntly.Font;
+import com.google.typography.font.sfntly.FontFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Simple WOFF 2.0 compression report runner.
+ *
+ * @author David Kuettel
+ */
+public class SimpleRunner {
+
+  private static final FontFactory FONT_FACTORY = FontFactory.getInstance();
+
+  private static final String GZIP = "gzip";
+  private static final String WOFF2 = "glyf/cbbox,triplet,code,reslice:woff2/lzma";
+
+  private static final String REPORT = "report.csv";
+
+  public static void main(String[] args) throws IOException {
+    if (args.length == 0) {
+      usage();
+    }
+    CompressionStats stats = new CompressionStats();
+
+    System.out.printf("Analyzing (%d) fonts\n", args.length);
+    run(stats, args);
+
+    System.out.printf("Creating report: %s\n", REPORT);
+    CsvReport.create(stats, REPORT);
+  }
+
+  private static void run(CompressionStats stats, String[] filenames) throws IOException {
+    for (String filename : filenames) {
+      File file = new File(filename);
+      byte[] bytes = Files.toByteArray(file);
+      Font font = FONT_FACTORY.loadFonts(bytes)[0];
+
+      byte[] gzip = Experiment.run(font, GZIP);
+      byte[] woff2 = Experiment.run(font, WOFF2);
+
+      stats.add(
+          CompressionStats.Stats.builder()
+              .setFilename(file.getName())
+              .setSize(CompressionStats.Size.ORIGINAL, bytes.length)
+              .setSize(CompressionStats.Size.GZIP, gzip.length)
+              .setSize(CompressionStats.Size.WOFF2, woff2.length)
+              .build());
+    }
+  }
+
+  private static void usage() {
+    System.err.println("Usage: SimpleRunner font...");
+    System.exit(-1);
+  }
+}