diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 4f7df64605dc..c758ee6bedbb 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -546,6 +546,58 @@ php_c: php-c-benchmark proto3_middleman_php ############ PHP RULES END ################# +############ protobuf.js RULE BEGIN ############# + +pbjs_preparation: + mkdir -p tmp/protobuf.js + cd tmp/protobuf.js && git clone https://github.com/dcodeIO/protobuf.js.git && \ + cd protobuf.js && npm install && npm run build + cd tmp/protobuf.js && npm install benchmark + cp protobuf.js/* tmp/protobuf.js + cp js/benchmark_suite.js tmp/protobuf.js + touch pbjs_preparation + +pbjs_middleman: pbjs_preparation + export OLDDIR=$$(pwd) && cd tmp/protobuf.js && node generate_pbjs_files.js --target static-module --include_path=$$OLDDIR -o generated_bundle_code.js $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) + touch pbjs_middleman + +pbjs-benchmark: pbjs_middleman + @echo '#! /bin/bash' > pbjs-benchmark + @echo 'cd tmp/protobuf.js' >> pbjs-benchmark + @echo 'sed -i "s/protobufjs/.\/protobuf.js/g" generated_bundle_code.js' >> pbjs-benchmark + @echo 'env NODE_PATH=".:./node_modules:$$NODE_PATH" node protobufjs_benchmark.js $$@' >> pbjs-benchmark + @chmod +x pbjs-benchmark + +pbjs: pbjs-benchmark + ./pbjs-benchmark $(all_data) + +############ protobuf.js RULE END ############# + +############ JS RULE BEGIN ############# + +js_preparation: + mkdir -p tmp/js + oldpwd=$$(pwd) && cd $(top_srcdir)/js && npm install && npm test + cd tmp/js && npm install benchmark + cp js/* tmp/js + touch js_preparation + +js_middleman: js_preparation + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --js_out=import_style=commonjs,binary:$$oldpwd/tmp/js $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2)) + touch js_middleman + +js-benchmark: js_middleman + @echo '#! /bin/bash' > js-benchmark + @echo 'export TOP_JS_SRCDIR=$$(cd $(top_srcdir)/js && pwd)' >> js-benchmark + @echo 'cd tmp/js' >> js-benchmark + @echo 'env NODE_PATH="$$TOP_JS_SRCDIR:.:./node_modules:$$NODE_PATH" node --max-old-space-size=4096 js_benchmark.js $$@' >> js-benchmark + @chmod +x js-benchmark + +js: js-benchmark + ./js-benchmark $(all_data) + +############ JS RULE END ############# + MAINTAINERCLEANFILES = \ Makefile.in @@ -593,8 +645,13 @@ CLEANFILES = \ generate_proto3_data \ php-benchmark \ php-c-benchmark \ - proto3_middleman_php - + proto3_middleman_php \ + pbjs_preparation \ + pbjs_middleman \ + pbjs-benchmark \ + js_preparation \ + js_middleman \ + js-benchmark clean-local: -rm -rf tmp/* diff --git a/benchmarks/README.md b/benchmarks/README.md index 91b80a923fa9..534e0bded71b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -62,6 +62,9 @@ The second command adds the `bin` directory to your `PATH` so that `protoc` can PHP benchmark's requirement is the same as PHP protobuf's requirements. The benchmark will automaticly include PHP protobuf's src and build the c extension if required. +### Node.js +Node.js benchmark need [node](https://nodejs.org/en/)(higher than V6) and [npm](https://www.npmjs.com/) package manager installed. This benchmark is using the [benchmark](https://www.npmjs.com/package/benchmark) framework to test, which needn't to manually install. And another prerequisite is [protobuf js](https://github.com/google/protobuf/tree/master/js), which needn't to manually install either + ### Big data There's some optional big testing data which is not included in the directory @@ -136,6 +139,11 @@ $ make php $ make php_c ``` +### Node.js +``` +$ make js +``` + To run a specific dataset or run with specific options: ### Java: @@ -195,6 +203,11 @@ $ make php-c-benchmark $ ./php-c-benchmark $(specific generated dataset file name) ``` +### Node.js +``` +$ make js-benchmark +$ ./js-benchmark $(specific generated dataset file name) +``` ## Benchmark datasets diff --git a/benchmarks/js/benchmark_suite.js b/benchmarks/js/benchmark_suite.js new file mode 100644 index 000000000000..c95024b2e244 --- /dev/null +++ b/benchmarks/js/benchmark_suite.js @@ -0,0 +1,33 @@ +var benchmark = require("benchmark"); + +function newBenchmark(messageName, filename, language) { + var benches = []; + return { + suite: new benchmark.Suite(messageName + filename + language ) + .on("add", function(event) { + benches.push(event.target); + }) + .on("start", function() { + process.stdout.write( + "benchmarking message " + messageName + + " of dataset file " + filename + + "'s performance ..." + "\n\n"); + }) + .on("cycle", function(event) { + process.stdout.write(String(event.target) + "\n"); + }) + .on("complete", function() { + var getHz = function(bench) { + return 1 / (bench.stats.mean + bench.stats.moe); + } + benches.forEach(function(val, index) { + benches[index] = getHz(val); + }); + }), + benches: benches + } +} + +module.exports = { + newBenchmark: newBenchmark +} diff --git a/benchmarks/js/js_benchmark.js b/benchmarks/js/js_benchmark.js new file mode 100644 index 000000000000..14905e312df6 --- /dev/null +++ b/benchmarks/js/js_benchmark.js @@ -0,0 +1,70 @@ +require('./datasets/google_message1/proto2/benchmark_message1_proto2_pb.js'); +require('./datasets/google_message1/proto3/benchmark_message1_proto3_pb.js'); +require('./datasets/google_message2/benchmark_message2_pb.js'); +require('./datasets/google_message3/benchmark_message3_pb.js'); +require('./datasets/google_message4/benchmark_message4_pb.js'); +require('./benchmarks_pb.js'); + +var fs = require('fs'); +var benchmarkSuite = require("./benchmark_suite.js"); + + +function getNewPrototype(name) { + var message = eval("proto." + name); + if (typeof(message) == "undefined") { + throw "type " + name + " is undefined"; + } + return message; +} + +var results = []; + +console.log("#####################################################"); +console.log("Js Benchmark: "); +process.argv.forEach(function(filename, index) { + if (index < 2) { + return; + } + var benchmarkDataset = + proto.benchmarks.BenchmarkDataset.deserializeBinary(fs.readFileSync(filename)); + var messageList = []; + var totalBytes = 0; + benchmarkDataset.getPayloadList().forEach(function(onePayload) { + var message = getNewPrototype(benchmarkDataset.getMessageName()); + messageList.push(message.deserializeBinary(onePayload)); + totalBytes += onePayload.length; + }); + + var senarios = benchmarkSuite.newBenchmark( + benchmarkDataset.getMessageName(), filename, "js"); + senarios.suite + .add("js deserialize", function() { + benchmarkDataset.getPayloadList().forEach(function(onePayload) { + var protoType = getNewPrototype(benchmarkDataset.getMessageName()); + protoType.deserializeBinary(onePayload); + }); + }) + .add("js serialize", function() { + var protoType = getNewPrototype(benchmarkDataset.getMessageName()); + messageList.forEach(function(message) { + message.serializeBinary(); + }); + }) + .run({"Async": false}); + + results.push({ + filename: filename, + benchmarks: { + protobufjs_decoding: senarios.benches[0] * totalBytes, + protobufjs_encoding: senarios.benches[1] * totalBytes + } + }) + + console.log("Throughput for deserialize: " + + senarios.benches[0] * totalBytes / 1024 / 1024 + "MB/s" ); + console.log("Throughput for serialize: " + + senarios.benches[1] * totalBytes / 1024 / 1024 + "MB/s" ); + console.log(""); +}); +console.log("#####################################################"); + diff --git a/benchmarks/protobuf.js/generate_pbjs_files.js b/benchmarks/protobuf.js/generate_pbjs_files.js new file mode 100644 index 000000000000..11945bf9e64b --- /dev/null +++ b/benchmarks/protobuf.js/generate_pbjs_files.js @@ -0,0 +1,25 @@ +var pbjs = require("./protobuf.js/cli").pbjs + +var argv = []; +var protoFiles = []; +var prefix = ""; +process.argv.forEach(function(val, index) { + var arg = val; + if (arg.length > 6 && arg.substring(arg.length - 6) == ".proto") { + protoFiles.push(arg); + } else if (arg.length > 15 && arg.substring(0, 15) == "--include_path=") { + prefix = arg.substring(15); + } else if (index >= 2) { + argv.push(arg); + } +}); +protoFiles.forEach(function(val) { + argv.push(prefix + "/" + val); +}); + +pbjs.main(argv, function(err, output){ + if (err) { + console.log(err); + } +}); + diff --git a/benchmarks/protobuf.js/protobufjs_benchmark.js b/benchmarks/protobuf.js/protobufjs_benchmark.js new file mode 100644 index 000000000000..2629e9f46f22 --- /dev/null +++ b/benchmarks/protobuf.js/protobufjs_benchmark.js @@ -0,0 +1,66 @@ +var root = require("./generated_bundle_code.js"); +var fs = require('fs'); +var benchmark = require("./node_modules/benchmark"); +var benchmarkSuite = require("./benchmark_suite.js"); + + +function getNewPrototype(name) { + var message = eval("root." + name); + if (typeof(message) == "undefined") { + throw "type " + name + " is undefined"; + } + return message; +} + + +var results = []; + +console.log("#####################################################"); +console.log("ProtobufJs Benchmark: "); +process.argv.forEach(function(filename, index) { + if (index < 2) { + return; + } + var benchmarkDataset = + root.benchmarks.BenchmarkDataset.decode(fs.readFileSync(filename)); + var messageList = []; + var totalBytes = 0; + benchmarkDataset.payload.forEach(function(onePayload) { + var message = getNewPrototype(benchmarkDataset.messageName); + messageList.push(message.decode(onePayload)); + totalBytes += onePayload.length; + }); + + var senarios = benchmarkSuite.newBenchmark( + benchmarkDataset.messageName, filename, "protobufjs"); + senarios.suite + .add("protobuf.js static decoding", function() { + benchmarkDataset.payload.forEach(function(onePayload) { + var protoType = getNewPrototype(benchmarkDataset.messageName); + protoType.decode(onePayload); + }); + }) + .add("protobuf.js static encoding", function() { + var protoType = getNewPrototype(benchmarkDataset.messageName); + messageList.forEach(function(message) { + protoType.encode(message).finish(); + }); + }) + .run({"Async": false}); + + results.push({ + filename: filename, + benchmarks: { + protobufjs_decoding: senarios.benches[0] * totalBytes, + protobufjs_encoding: senarios.benches[1] * totalBytes + } + }) + + console.log("Throughput for decoding: " + + senarios.benches[0] * totalBytes / 1024 / 1024 + "MB/s" ); + console.log("Throughput for encoding: " + + senarios.benches[1] * totalBytes / 1024 / 1024 + "MB/s" ); + console.log(""); +}); +console.log("#####################################################"); +