Skip to content

Commit

Permalink
System attributes md5 for sqs (#3014)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mullermp
mullermp committed Apr 29, 2024
1 parent 3ff1eda commit fd7984d
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 40 deletions.
2 changes: 2 additions & 0 deletions gems/aws-sdk-sqs/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
Unreleased Changes
------------------

* Feature - Handle System Message Attributes MD5 verification.

1.71.0 (2024-04-25)
------------------

Expand Down
119 changes: 84 additions & 35 deletions gems/aws-sdk-sqs/lib/aws-sdk-sqs/plugins/md5s.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ module Aws
module SQS
module Plugins
class Md5s < Seahorse::Client::Plugin

# @api private
class Handler < Seahorse::Client::Handler
def call(context)
Expand All @@ -26,80 +25,105 @@ def call(context)
'String' => 1,
'Binary' => 2,
'Number' => 1
}
}.freeze

DATA_TYPE = /\A(String|Binary|Number)(\..+)?\z/
DATA_TYPE = /\A(String|Binary|Number)(\..+)?\z/.freeze

NORMALIZED_ENCODING = Encoding::UTF_8

def validate_send_message(context, response)
body = context.params[:message_body]
attributes = context.params[:message_attributes]
validate_single_message(body, attributes, response)
system_attributes = context.params[:message_system_attributes]
validate_single_message(body, attributes, system_attributes, response)
end

def validate_send_message_batch(context, response)
context.params[:entries].each do |entry|
id = entry[:id]
body = entry[:message_body]
attributes = entry[:message_attributes]
system_attributes = entry[:message_system_attributes]
message_response = response.successful.select { |r| r.id == id }[0]
unless message_response.nil?
validate_single_message(body, attributes, message_response)
validate_single_message(body, attributes, system_attributes, message_response)
end
end
end

def validate_single_message(body, attributes, response)
def validate_single_message(body, attributes, system_attributes, response)
validate_body(body, response)
unless attributes.nil? || attributes.empty?
validate_attributes(attributes, response)
end
unless system_attributes.nil? || system_attributes.empty?
validate_system_attributes(system_attributes, response)
end
end

def validate_body(body, response)
calculated_md5 = md5_of_message_body(body)
returned_md5 = response.md5_of_message_body
if calculated_md5 != returned_md5
error_message = mismatch_error_message(
'message body',
calculated_md5,
returned_md5,
response)
raise Aws::Errors::ChecksumError, error_message
end
return unless calculated_md5 != returned_md5

error_message = mismatch_error_message(
'message body',
calculated_md5,
returned_md5,
response
)
raise Aws::Errors::ChecksumError, error_message
end

def validate_attributes(attributes, response)
calculated_md5 = md5_of_message_attributes(attributes)
returned_md5 = response.md5_of_message_attributes
if returned_md5 != calculated_md5
error_message = mismatch_error_message(
'message attributes',
calculated_md5,
returned_md5,
response)
raise Aws::Errors::ChecksumError, error_message
end
return unless returned_md5 != calculated_md5

error_message = mismatch_error_message(
'message attributes',
calculated_md5,
returned_md5,
response
)
raise Aws::Errors::ChecksumError, error_message
end

def validate_system_attributes(system_attributes, response)
calculated_md5 = md5_of_message_system_attributes(system_attributes)
returned_md5 = response.md5_of_message_system_attributes
return unless returned_md5 != calculated_md5

error_message = mismatch_error_message(
'message system attributes',
calculated_md5,
returned_md5,
response
)
raise Aws::Errors::ChecksumError, error_message
end

def md5_of_message_body(message_body)
OpenSSL::Digest::MD5.hexdigest(message_body)
end

# MD5 of Message Attributes and System Attributes are effectively
# the same calculation. However, keeping these as two methods because
# they are modeled as two different shapes.
###
def md5_of_message_attributes(message_attributes)
encoded = { }
encoded = {}
message_attributes.each do |name, attribute|
name = name.to_s
encoded[name] = String.new
data_type_without_label = DATA_TYPE.match(attribute[:data_type])[1]
encoded[name] << encode_length_and_bytes(name) <<
encode_length_and_bytes(attribute[:data_type]) <<
[TRANSPORT_TYPE_ENCODINGS[data_type_without_label]].pack('C'.freeze)
encode_length_and_bytes(attribute[:data_type]) <<
[TRANSPORT_TYPE_ENCODINGS[data_type_without_label]].pack('C')

if attribute[:string_value] != nil
if !attribute[:string_value].nil?
encoded[name] << encode_length_and_string(attribute[:string_value])
elsif attribute[:binary_value] != nil
elsif !attribute[:binary_value].nil?
encoded[name] << encode_length_and_bytes(attribute[:binary_value])
end
end
Expand All @@ -110,14 +134,38 @@ def md5_of_message_attributes(message_attributes)
OpenSSL::Digest::MD5.hexdigest(buffer)
end

def md5_of_message_system_attributes(message_system_attributes)
encoded = {}
message_system_attributes.each do |name, attribute|
name = name.to_s
encoded[name] = String.new
data_type_without_label = DATA_TYPE.match(attribute[:data_type])[1]
encoded[name] << encode_length_and_bytes(name) <<
encode_length_and_bytes(attribute[:data_type]) <<
[TRANSPORT_TYPE_ENCODINGS[data_type_without_label]].pack('C')

if !attribute[:string_value].nil?
encoded[name] << encode_length_and_string(attribute[:string_value])
elsif !attribute[:binary_value].nil?
encoded[name] << encode_length_and_bytes(attribute[:binary_value])
end
end

buffer = encoded.keys.sort.reduce(String.new) do |string, name|
string << encoded[name]
end
OpenSSL::Digest::MD5.hexdigest(buffer)
end
###

def encode_length_and_string(string)
string = String.new(string)
string.encode!(NORMALIZED_ENCODING)
encode_length_and_bytes(string)
end

def encode_length_and_bytes(bytes)
[bytes.bytesize, bytes].pack('L>a*'.freeze)
[bytes.bytesize, bytes].pack('L>a*')
end

def mismatch_error_message(section, local_md5, returned_md5, response)
Expand Down Expand Up @@ -154,13 +202,14 @@ def mismatch_error_message(section, local_md5, returned_md5, response)
end

def add_handlers(handlers, config)
if config.verify_checksums
handlers.add(Handler, {
priority: 10 ,
step: :validate,
operations: [:send_message, :send_message_batch]
})
end
return unless config.verify_checksums

handlers.add(
Handler,
priority: 10,
step: :validate,
operations: %i[send_message send_message_batch]
)
end
end
end
Expand Down
46 changes: 41 additions & 5 deletions gems/aws-sdk-sqs/spec/client/verify_checksums_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,25 @@ class Client
}
}}

let(:message_system_attributes) {{
'ccc' => {
string_value: 'test',
data_type: 'String'
},
aaa: {
binary_value: [ 2, 3, 4 ].pack('C*'),
data_type: 'Binary'
},
zzz: {
data_type: 'Number',
string_value: '0230.01'
},
'öther_encodings' => {
data_type: 'String',
string_value: 'Tüst'.encode('ISO-8859-1')
}
}}

it 'is enabled by default' do
client = Client.new(
region: 'us-east-1',
Expand All @@ -52,13 +71,15 @@ class Client

describe '#send_message' do
let(:md5_of_attributes) { '756d7f4338696745d063b420a2f7e502' }
let(:md5_of_system_attributes) { '756d7f4338696745d063b420a2f7e502' }

before(:each) do
response_body = <<-JSON
{
"MD5OfMessageAttributes": "#{md5_of_attributes}",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MessageId": "5fea7756-0ea4-451a-a703-a558b933e274"
"MessageId": "5fea7756-0ea4-451a-a703-a558b933e274",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MD5OfMessageAttributes": "#{md5_of_attributes}",
"MD5OfMessageSystemAttributes": "#{md5_of_system_attributes}"
}
JSON

Expand All @@ -77,6 +98,7 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.not_to raise_error
end
Expand All @@ -87,17 +109,22 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: {},
message_system_attributes: {}
)
}.not_to raise_error
end

context 'when data types have custom labels' do
let(:md5_of_attributes) { '5b7ef6c8a8d46001c7cdadaeea917aa4' }
let(:md5_of_system_attributes) { '5b7ef6c8a8d46001c7cdadaeea917aa4' }

before(:each) do
message_attributes.keys.each do |attribute_name|
message_attributes[attribute_name][:data_type] += '.test'
end
message_system_attributes.keys.each do |attribute_name|
message_system_attributes[attribute_name][:data_type] += '.test'
end
end

it 'does not raise an error if checksums match' do
Expand All @@ -106,6 +133,7 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.not_to raise_error
end
Expand All @@ -117,17 +145,20 @@ class Client
queue_url:'https://queue.url',
message_body: message_body + 'junk',
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::Errors::ChecksumError)
end

it 'raises when the md5 checksums do not match for the body' do
message_attributes['ccc'][:string_value] += 'junk'
message_system_attributes['ccc'][:string_value] += 'junk'
expect {
client.send_message(
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::Errors::ChecksumError)
end
Expand All @@ -146,14 +177,14 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::SQS::Errors::ServiceError)
end

end

describe '#send_message_batch' do

before(:each) do
client.handle(step: :send) do |context|
context.http_response.signal_done(
Expand All @@ -165,7 +196,8 @@ class Client
{
"Id": "msg-id",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MD5OfMessageAttributes": "756d7f4338696745d063b420a2f7e502"
"MD5OfMessageAttributes": "756d7f4338696745d063b420a2f7e502",
"MD5OfMessageSystemAttributes": "756d7f4338696745d063b420a2f7e502"
}
]
}
Expand All @@ -183,6 +215,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand All @@ -198,6 +231,7 @@ class Client
id: 'msg-id',
message_body: message_body + 'junk',
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand All @@ -214,6 +248,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand Down Expand Up @@ -245,6 +280,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand Down

0 comments on commit fd7984d

Please sign in to comment.