Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
FEATURE: support custom codecs for transport (#250)
Adds support for MessageBus.transport_codec which can be used to optimise performance of transport - Oj can be used as a serializer vs JSON which is a bit slower - Users can implement custom codecs
- Loading branch information
1 parent
4dabfdc
commit 0d79789
Showing
18 changed files
with
342 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,7 @@ | ||
UNRELEASED | ||
|
||
- FEATURE: Introduce support for transport codecs | ||
|
||
28-04-2021 | ||
|
||
- Version 3.3.5 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# frozen_string_literal: true | ||
|
||
require_relative './packed_string' | ||
require_relative './string_hack' | ||
require_relative './marshal' | ||
|
||
def all_codecs | ||
{ | ||
json: MessageBus::Codec::Json.new, | ||
oj: MessageBus::Codec::Oj.new, | ||
marshal: MarshalCodec.new, | ||
packed_string_4_bytes: PackedString.new("V"), | ||
packed_string_8_bytes: PackedString.new("Q"), | ||
string_hack: StringHack.new | ||
} | ||
end | ||
|
||
def bench_decode(hash, user_needle) | ||
encoded_data = all_codecs.map do |name, codec| | ||
[ | ||
name, codec, codec.encode(hash.dup) | ||
] | ||
end | ||
|
||
Benchmark.ips do |x| | ||
|
||
encoded_data.each do |name, codec, encoded| | ||
x.report(name) do |n| | ||
while n > 0 | ||
decoded = codec.decode(encoded) | ||
decoded["user_ids"].include?(user_needle) | ||
n -= 1 | ||
end | ||
end | ||
end | ||
|
||
x.compare! | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# frozen_string_literal: true | ||
|
||
class MarshalCodec | ||
def encode(hash) | ||
::Marshal.dump(hash) | ||
end | ||
|
||
def decode(payload) | ||
::Marshal.load(payload) | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
# frozen_string_literal: true | ||
|
||
class PackedString | ||
class FastIdList | ||
def self.from_array(array, pack_with) | ||
new(array.sort.pack("#{pack_with}*"), pack_with) | ||
end | ||
|
||
def self.from_string(string, pack_with) | ||
new(string, pack_with) | ||
end | ||
|
||
def initialize(packed, pack_with) | ||
raise "unknown pack format, expecting Q or V" if pack_with != "V" && pack_with != "Q" | ||
@packed = packed | ||
@pack_with = pack_with | ||
@slot_size = pack_with == "V" ? 4 : 8 | ||
end | ||
|
||
def include?(id) | ||
found = (0...length).bsearch do |index| | ||
@packed.byteslice(index * @slot_size, @slot_size).unpack1(@pack_with) >= id | ||
end | ||
|
||
found && @packed.byteslice(found * @slot_size, @slot_size).unpack1(@pack_with) == id | ||
end | ||
|
||
def length | ||
@length ||= @packed.bytesize / @slot_size | ||
end | ||
|
||
def to_a | ||
@packed.unpack("#{@pack_with}*") | ||
end | ||
|
||
def to_s | ||
@packed | ||
end | ||
end | ||
|
||
def initialize(pack_with = "V") | ||
@pack_with = pack_with | ||
@oj_options = { mode: :compat } | ||
end | ||
|
||
def encode(hash) | ||
|
||
if user_ids = hash["user_ids"] | ||
hash["user_ids"] = FastIdList.from_array(hash["user_ids"], @pack_with).to_s | ||
end | ||
|
||
hash["data"] = ::Oj.dump(hash["data"], @oj_options) | ||
|
||
Marshal.dump(hash) | ||
end | ||
|
||
def decode(payload) | ||
result = Marshal.load(payload) | ||
result["data"] = ::Oj.load(result["data"], @oj_options) | ||
|
||
if str = result["user_ids"] | ||
result["user_ids"] = FastIdList.from_string(str, @pack_with) | ||
end | ||
|
||
result | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# frozen_string_literal: true | ||
|
||
class StringHack | ||
class FastIdList | ||
def self.from_array(array) | ||
new(",#{array.join(",")},") | ||
end | ||
|
||
def self.from_string(string) | ||
new(string) | ||
end | ||
|
||
def initialize(packed) | ||
@packed = packed | ||
end | ||
|
||
def include?(id) | ||
@packed.include?(",#{id},") | ||
end | ||
|
||
def to_s | ||
@packed | ||
end | ||
end | ||
|
||
def initialize | ||
@oj_options = { mode: :compat } | ||
end | ||
|
||
def encode(hash) | ||
if user_ids = hash["user_ids"] | ||
hash["user_ids"] = FastIdList.from_array(user_ids).to_s | ||
end | ||
|
||
::Oj.dump(hash, @oj_options) | ||
end | ||
|
||
def decode(payload) | ||
result = ::Oj.load(payload, @oj_options) | ||
|
||
if str = result["user_ids"] | ||
result["user_ids"] = FastIdList.from_string(str) | ||
end | ||
|
||
result | ||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
# frozen_string_literal: true | ||
|
||
require 'bundler/inline' | ||
|
||
gemfile do | ||
source 'https://rubygems.org' | ||
gem 'message_bus', path: '../' | ||
gem 'benchmark-ips' | ||
gem 'oj' | ||
end | ||
|
||
require 'benchmark/ips' | ||
require 'message_bus' | ||
require_relative 'codecs/all_codecs' | ||
|
||
bench_decode({ | ||
"data" => "hello world", | ||
"user_ids" => (1..10000).to_a, | ||
"group_ids" => nil, | ||
"client_ids" => nil | ||
}, 5000 | ||
) | ||
|
||
# packed_string_4_bytes: 127176.1 i/s | ||
# packed_string_8_bytes: 94494.6 i/s - 1.35x (± 0.00) slower | ||
# string_hack: 26403.4 i/s - 4.82x (± 0.00) slower | ||
# marshal: 4985.5 i/s - 25.51x (± 0.00) slower | ||
# oj: 3072.9 i/s - 41.39x (± 0.00) slower | ||
# json: 2222.7 i/s - 57.22x (± 0.00) slower |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
# frozen_string_literal: true | ||
|
||
require 'bundler/inline' | ||
|
||
gemfile do | ||
source 'https://rubygems.org' | ||
gem 'message_bus', path: '../' | ||
gem 'benchmark-ips' | ||
gem 'oj' | ||
end | ||
|
||
require 'benchmark/ips' | ||
require 'message_bus' | ||
require_relative 'codecs/all_codecs' | ||
|
||
bench_decode({ | ||
"data" => { amazing: "hello world this is an amazing message hello there!!!", another_key: [2, 3, 4] }, | ||
"user_ids" => [1, 2, 3], | ||
"group_ids" => [1], | ||
"client_ids" => nil | ||
}, 2 | ||
) | ||
|
||
# marshal: 504885.6 i/s | ||
# json: 401050.9 i/s - 1.26x (± 0.00) slower | ||
# oj: 340847.4 i/s - 1.48x (± 0.00) slower | ||
# string_hack: 296741.6 i/s - 1.70x (± 0.00) slower | ||
# packed_string_4_bytes: 207942.6 i/s - 2.43x (± 0.00) slower | ||
# packed_string_8_bytes: 206093.0 i/s - 2.45x (± 0.00) slower |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.