Restructuring to provide per-serializer classloading.

tpc/Makefile

@@ -14,7 +14,7 @@
 
 # Need 'javac' version 5 or greater.
 JavaC ?= javac
-JavaC_Opt := -Xlint:unchecked
+JavaC_Opt := -Xlint:unchecked -g
 
 empty :=
 space := $(empty) $(empty)
@@ -28,7 +28,7 @@ protobuf_generator ?= protoc
 thrift_generator ?= thrift
 activemqprotobuf_cp := $(call mkcp,$(wildcard lib/activemq-*.jar))
 activemqprotobuf_generator ?= java -cp "$(activemqprotobuf_cp)" org.apache.activemq.protobuf.compiler.AltJavaGenerator
-cks_generator ?= java -jar lib/cks-tool.jar
+cks_generator ?= java -cp lib-core/cks-core.jar:lib-core/cks-text-reader.jar:lib-core/cks-text-writer.jar -jar lib-core/cks-tool.jar
 avro_cp ?= $(call mkcp,$(wildcard lib/avro-*.jar lib/jackson-*.jar lib/jopt-simple-*.jar lib/velocity-*.jar lib/slf4j-*.jar))
 avro_generator ?= java -cp "$(avro_cp)" org.apache.avro.tool.Main
 protostuff_compiler ?= $(wildcard lib/protostuff-compiler-*.jar)
@@ -47,7 +47,7 @@ Generate_protostuff ?= y
 
 .PHONY: default compile
 .DELETE_ON_ERROR:
-default: compile build/gen-cp build/lib-cp
+default: compile build/gen-cp build/lib-cp build/lib-core-cp
 
 
 p1 := @
@@ -76,6 +76,9 @@ build/lib-cp: lib
 	$(p2)mkdir -p build
 	$(p1)echo "$(call mkcp,$(wildcard lib/*.jar))" > "$@"
 
+build/lib-core-cp: build/lib-cp
+	$(p1)echo "$(call mkcp,$(wildcard lib-core/*.jar))" > "$@"
+
 ##########################################################################
 # Generate and Compile Bindings
 # $(1): name, $(2): filename
@@ -138,14 +141,14 @@ JavaSourceFiles_$(1) = $$(shell find $$(JavaSourceRoots_$(1)) -type f -name '*.j
 endif
 
 # Compile to bytecode
-build/bytecode/gen/$(2).compile.$$(StampSuffix_$(1)): $$(JavaSourceRoots_$(1)) $$(JavaSourceDirs_$(1)) $$(JavaSourceFiles_$(1)) lib build/lib-cp $(wildcard lib/*.jar)
+build/bytecode/gen/$(2).compile.$$(StampSuffix_$(1)): $$(JavaSourceRoots_$(1)) $$(JavaSourceDirs_$(1)) $$(JavaSourceFiles_$(1)) lib build/lib-cp build/lib-core-cp $(wildcard lib/*.jar)
 	$(p2)[ ! -e build/bytecode/gen/$(2) ] || rm -r build/bytecode/gen/$(2)
 	$(p2)mkdir -p build/bytecode/gen/$(2)
 	$(p2)rm -f build/bytecode/gen/$(2).compile.*
 	# Compile: $$(JavaSourceRoots_$(1))
 	$(p2)touch "$$@"
 	$(p1)$(JavaC) $(JavaC_Opt) $$(JavaC_Opt_$(2)) -d build/bytecode/gen/$(2) \
-		-classpath "$$(shell cat build/lib-cp)" \
+		-classpath "$$(shell cat build/lib-cp) $$(shell cat build/lib-core-cp)" \
 		$$(shell find $$(JavaSourceRoots_$(1)) -type f -name '*.java')
 
 build/bytecode/gen/$(2).compile: build/bytecode/gen/$(2).compile.$$(StampSuffix_$(1))
@@ -165,7 +168,7 @@ JavaSourceDirs = $(shell find $(JavaSourceRoots) -type d)
 JavaSourceFiles = $(shell find $(JavaSourceRoots) -type f -name '*.java')
 ScalaSourceFiles = $(shell find $(JavaSourceRoots) -type f -name '*.scala')
 
-build/bytecode/main.compile: $(JavaSourceRoots) $(JavaSourceDirs) $(JavaSourceFiles) $(ScalaSourceFiles) lib build/lib-cp build/gen-cp $(wildcard lib/*.jar) $(Schemas:%=build/bytecode/gen/%.compile)
+build/bytecode/main.compile: $(JavaSourceRoots) $(JavaSourceDirs) $(JavaSourceFiles) $(ScalaSourceFiles) lib build/lib-cp build/lib-core-cp build/gen-cp $(wildcard lib/*.jar) $(Schemas:%=build/bytecode/gen/%.compile)
 	$(p2)[ ! -e build/bytecode/main ] || rm -r build/bytecode/main
 	$(p2)mkdir -p build/bytecode/main
 	$(p2)touch "$@"
@@ -180,11 +183,15 @@ else
 	# Compile: $(JavaSourceRoots) [javac]
 	$(p1)$(JavaC) $(JavaC_Opt) -d build/bytecode/main \
 		-deprecation \
-		-classpath "$(call mkcp,build/bytecode/main $(shell cat build/lib-cp) $(shell cat build/gen-cp))" \
+		-classpath "$(call mkcp,build/bytecode/main $(shell cat build/lib-cp) $(shell cat build/gen-cp) $(shell cat build/lib-core-cp))" \
 		$(JavaSourceFiles)
 endif
 
 compile: build/bytecode/main.compile
+	$(p2)rm -rf build/bytecode/main/sx
+	$(p2)mkdir -p build/bytecode/main/sx
+	$(p2)mv build/bytecode/main/serializers build/bytecode/main/sx/
+	# $(p2)mv build/bytecode/main/ext build/bytecode/main/sx/
 
 ##########################################################################
 # Clean

tpc/chart.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+# This script produces the equivalent charts using the data output by running
+# the benchmark. Just add your output lines to the end of this file and run it.
+# Take a look below for an example...
+
+require 'uri'
+
+TITLES = [ 
+    "create (nanos)",
+    "ser (nanos)",
+    "ser+same (nanos)",
+    "deser (nanos)",
+    "deser+shal (nanos)",
+    "deser+deep (nanos)",
+    "total (nanos)",
+    "size (bytes)",
+    "size+dfl (bytes)"
+]
+
+data = {}
+names = []
+
+name_str = ""
+DATA.readlines.each do |x|
+    x.chomp!
+    next if x =~ /^#/ or x.empty?
+    y = x.split
+    name = y.shift
+    data[name] = y.collect { |v| v.to_i }
+    names << name
+    name_str = "#{name}|#{name_str}"
+end
+
+name_str = name_str[0..-2]
+name_str = URI.encode_www_form_component(name_str)
+
+idx = 0
+width = 500
+height = names.size * 20 + 30
+bar_thickness = 10
+bar_spacing = 10
+TITLES.each do |title|
+    max = 0
+
+    val_str = ""
+    names.each do |n|
+        max = data[n][idx] if data[n][idx] > max 
+        val_str += "#{data[n][idx]},"
+    end
+
+    val_str = val_str[0..-2]
+    scale = max * 1.1
+
+    puts "<img src='https://chart.googleapis.com/chart?chtt=#{URI.encode_www_form_component(title)}&chf=c||lg||0||FFFFFF||1||76A4FB||0|bg||s||EFEFEF&chs=#{width}x#{height}&chd=t:#{val_str}&chds=0,#{scale}&chxt=y&chxl=0:|#{name_str}&chm=N *f*,000000,0,-1,10&lklk&chdlp=t&chco=660000|660033|660066|660099|6600CC|6600FF|663300|663333|663366|663399|6633CC|6633FF|666600|666633|666666&cht=bhg&chbh=#{bar_thickness},0,#{bar_spacing}&nonsense=aaa.png'/>"
+
+    idx += 1
+end
+
+
+__END__
+#                                create     ser   +same   deser   +shal   +deep   total   size  +dfl
+java-built-in                       213   11785   10656   50080   50026   50766   62551    889   514
+bson/mongodb                        138    7815    7829   30058   30484   30622   38437    495   278
+avro                                173    4380    4158    2058    2815    3839    8219    221   133
+protobuf                            450    4197    1874    1697    1742    2253    6449    239   149

tpc/run

@@ -6,8 +6,9 @@ case "`uname`" in
 	CYGWIN*) path_sep=";" ;;
 esac
 
-cp=build/bytecode/main$path_sep$(cat build/gen-cp)$path_sep$(cat build/lib-cp)
-exec java -Xmx16m -server -cp "$cp" serializers.BenchmarkRunner "$@"
+cp=build/bytecode/main$path_sep$(cat build/lib-core-cp)
+loaderClasspath=build/bytecode/main/sx$path_sep$(cat build/lib-cp)$path_sep$(cat build/gen-cp)
+exec java -Dgemfire.AutoSerializer.SAFE=true -Xmx16m -server -cp "$cp" -Dloader.classpath=${loaderClasspath} core.BenchmarkRunner "$@"
 
 # Values used during testing are small.  Should probably use fairly
 # small heap to capture effects of memory allocation

tpc/run-stream

@@ -7,4 +7,4 @@ case "`uname`" in
 esac
 
 cp=build/bytecode/main$path_sep$(cat build/gen-cp)$path_sep$(cat build/lib-cp)
-exec java -Xmx64m -server -cp "$cp" serializers.MediaStreamBenchmark "$@"
+exec java -Xmx64m -server -cp "$cp" core.MediaStreamBenchmark "$@"

tpc/src/serializers/BenchmarkBase.java → tpc/src/core/BenchmarkBase.java

@@ -1,6 +1,12 @@
-package serializers;
+package core;
+
+import core.serializers.Serializer;
 
 import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.net.URLEncoder;
 import java.util.*;
 import java.util.regex.Pattern;
@@ -14,6 +20,8 @@ abstract class BenchmarkBase
     public final static int DEFAULT_ITERATIONS = 2000;
     public final static int DEFAULT_TRIALS = 20;
 
+    private List<URL> classPathUrls = new ArrayList<URL>();
+
     /**
      * Number of milliseconds to warm up for each operation type for each serializer. Let's
      * start with 3 seconds.
@@ -28,6 +36,90 @@ abstract class BenchmarkBase
             HIDDEN.add("cks-text");
     }
 
+    /*
+     * This map holds the mapping of the name of the test (benchmark) and the
+     * class which implements it. This is so that the tests can be instantiated
+     * within their own classloaders in order to provide some separation.
+     */
+    protected static final Map<String, String> testClassMap =
+            new HashMap<String, String>();
+    static {
+        // Binary Formats; language-specific ones
+        testClassMap.put("java-built-in", "serializers.JavaBuiltIn");
+
+        testClassMap.put("java-manual", "serializers.JavaManual");
+        testClassMap.put("scala/sbinary", "serializers.Scala");
+//        // hessian, kryo and wobly are Java object serializations
+//        Hessian.register(groups);
+//        Kryo.register(groups);
+//        Wobly.register(groups);
+
+//        Binary formats, generic: protobuf, thrift, avro, CKS, msgpack
+        testClassMap.put("protobuf", "serializers.protobuf.Protobuf");
+//        ActiveMQProtobuf.register(groups);
+        testClassMap.put("protostuff", "serializers.protostuff.Protostuff");
+        testClassMap.put("thrift", "serializers.Thrift");
+        testClassMap.put("avro", "serializers.AvroSpecific");
+        testClassMap.put("avro-generic", "serializers.AvroGeneric");
+//        CksBinary.register(groups);
+//        MsgPack.register(groups);
+//
+//        // JSON
+        testClassMap.put("json/jackson/manual", "serializers.jackson.JacksonJsonManual");
+//        JacksonJsonDatabind.register(groups);
+//        JacksonJsonDatabindWithStrings.register(groups);
+//        JacksonJsonAfterburner.register(groups); // databind with bytecode generation (faster)
+//        JacksonJsonTree.register(groups);
+//        JacksonJsonTreeWithStrings.register(groups);
+//        JsonTwoLattes.register(groups);
+//        ProtostuffJson.register(groups);
+//// too slow, why bother:
+////        ProtobufJson.register(groups);
+//        JsonGsonManual.register(groups);
+//        JsonGsonTree.register(groups);
+//        JsonGsonDatabind.register(groups);
+//        JsonSvensonDatabind.register(groups);
+//        FlexjsonDatabind.register(groups);
+//        JsonLibJsonDatabind.register(groups);
+//        FastJSONDatabind.register(groups);
+//        JsonSimpleWithContentHandler.register(groups);
+//        JsonSimpleManualTree.register(groups);
+//        JsonSmartManualTree.register(groups);
+//        JsonDotOrgManualTree.register(groups);
+//        JsonijJpath.register(groups);
+//        JsonijManualTree.register(groups);
+//        JsonArgoTree.register(groups);
+//        JsonPathDeserializerOnly.register(groups);
+//
+//        // Then JSON-like
+//        // CKS text is textual JSON-like format
+        testClassMap.put("cks-text", "serializers.cks.CksText");
+
+//        // then binary variants
+//        // Smile is 1-to-1 binary JSON serialization
+        testClassMap.put("smile/jackson/manual", "serializers.jackson.JacksonSmileManual");
+//        JacksonSmileManual.register(groups);
+//        JacksonSmileDatabind.register(groups);
+//        JacksonSmileAfterburner.register(groups); // databind with bytecode generation (faster)
+//        ProtostuffSmile.register(groups);
+//        // BSON is JSON-like format with extended datatypes
+//        JacksonBsonManual.register(groups);
+//        JacksonBsonDatabind.register(groups);
+//        MongoDB.register(groups);
+        testClassMap.put("bson/mongodb", "serializers.MongoDB");
+//
+//        // XML-based formats.
+//        XmlStax.register(groups, true, true, true); // woodstox/aalto/fast-infoset
+//        XmlXStream.register(groups);
+//        JacksonXmlDatabind.register(groups);
+//        XmlJavolution.register(groups);
+
+        // Gemfire
+//        testClassMap.put("gemfire/pdx-auto", "serializers.GemFirePdxAuto");
+//        testClassMap.put("gemfire/pdx-serializer", "serializers.GemfirePdxSerializer");
+//        testClassMap.put("gemfire/pdx-serializable", "serializers.GemfirePdxSerializable");
+//        testClassMap.put("gemfire/data-serializable", "serializers.GemfireDataSerializable");
+    }
     protected static final String ERROR_DIVIDER = "-------------------------------------------------------------------";
 
     // ------------------------------------------------------------------------------------
@@ -67,7 +159,21 @@ protected final static class Params
         public String dataExtra; // from second part
         public String dataExtension; // from last part of file name
     }
-
+
+    protected BenchmarkBase() throws IOException {
+        String classPath = System.getProperty("loader.classpath");
+        String pwd = new File(".").getCanonicalPath();
+        for (String s : classPath.split(":")) {
+            if (! (s.endsWith(".jar") || s.endsWith(".war") || s.endsWith(".ear"))) {
+                s += "/";
+            }
+            if (s.startsWith("/")) {
+                classPathUrls.add(new URL("file://" + s));
+            } else {
+                classPathUrls.add(new URL("file://" + pwd + "/" + s));
+            }
+        }
+    }
 
     // ------------------------------------------------------------------------------------
     // Actual benchmark flow
@@ -76,12 +182,14 @@ protected final static class Params
     protected void runBenchmark(String[] args,
             TestCase testCreate,
             TestCase testSerialize, TestCase testSerializeSameObject,
-            TestCase testDeserialize, TestCase testDeserializeAndCheck, TestCase testDeserializeAndCheckShallow)
-        {
+            TestCase testDeserialize, TestCase testDeserializeAndCheck, TestCase testDeserializeAndCheckShallow) throws Exception {
         Params params = new Params();
         findParameters(args, params);
         TestGroups groups = new TestGroups();
-        addTests(groups);
+
+        // We always need cks
+        params.filterStrings.add("cks-text");
+        addTests(groups, params.filterStrings);
         runTests(groups, params,
                 testCreate,
                 testSerialize, testSerializeSameObject,
@@ -91,7 +199,7 @@ protected void runBenchmark(String[] args,
     /**
      * Method called to find add actual test codecs
      */
-    protected abstract void addTests(TestGroups groups);
+    protected abstract void addTests(TestGroups groups, Set<String> tests) throws Exception;
 
     protected void findParameters(String[] args, Params params)
     {
@@ -574,12 +682,14 @@ protected static void addValue(
 
     protected <J> void warmTest(TestCaseRunner<J> runner, long warmupTime, TestCase test) throws Exception
     {
-        // Instead of fixed counts, let's try to prime by running for N seconds
-        long endTime = System.currentTimeMillis() + warmupTime;
-        do {
-            runner.run(test, 10);
+        if (warmupTime > 0) {
+            // Instead of fixed counts, let's try to prime by running for N seconds
+            long endTime = System.currentTimeMillis() + warmupTime;
+            do {
+                runner.run(test, 10);
+            }
+            while (System.currentTimeMillis() < endTime);
         }
-        while (System.currentTimeMillis() < endTime);
     }
 
     // ------------------------------------------------------------------------------------
@@ -741,7 +851,21 @@ protected static void printImage(Map<String, Double> map, measurements m)
             System.err.println("WARNING: Not enough room to fit all bars in chart.");
         }
     }
-
+
+    @SuppressWarnings("unchecked")
+    protected void register(String className, TestGroups groups)
+            throws Exception {
+
+        ClassLoader loader = newLoader();
+        Class c = loader.loadClass(className);
+        Method m = c.getDeclaredMethod("register", TestGroups.class);
+        m.invoke(null, groups);
+    }
+
+    private ClassLoader newLoader() {
+        return new URLClassLoader(classPathUrls.toArray(new URL[]{}),
+                BenchmarkRunner.class.getClassLoader());
+    }
     // ------------------------------------------------------------------------------------
     // Static helper methods
     // ------------------------------------------------------------------------------------
@@ -767,13 +891,13 @@ protected static void doGc()
         try {
             Thread.sleep(50L);
         } catch (InterruptedException ie) {
-            System.err.println("Interrupted while sleeping in serializers.BenchmarkBase.doGc()");
+            System.err.println("Interrupted while sleeping in core.BenchmarkBase.doGc()");
         }
         System.gc();
         try { // longer sleep afterwards (not needed by GC, but may help with scheduling)
             Thread.sleep(200L);
         } catch (InterruptedException ie) {
-            System.err.println("Interrupted while sleeping in serializers.BenchmarkBase.doGc()");
+            System.err.println("Interrupted while sleeping in core.BenchmarkBase.doGc()");
         }
     }