Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

System attributes md5 for sqs #3014

Merged
merged 5 commits into from
Apr 29, 2024
Merged

System attributes md5 for sqs #3014

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fd2554f
System attributes md5 for sqs and open up verify checksum with testing
mullermp Apr 29, 2024
4cf966c
Revert change to testing md5 in sqs
mullermp Apr 29, 2024
90edece
Actually revert sqs config change
mullermp Apr 29, 2024
0d0bbe8
Add changelog
mullermp Apr 29, 2024
2d9aa01
PR feedback + rubocop
mullermp Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions gems/aws-sdk-sqs/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
Unreleased Changes
------------------

* Feature - Handle System Message Attributes MD5 verification.

1.71.0 (2024-04-25)
------------------

Expand Down
52 changes: 49 additions & 3 deletions gems/aws-sdk-sqs/lib/aws-sdk-sqs/plugins/md5s.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,31 @@ def call(context)
def validate_send_message(context, response)
body = context.params[:message_body]
attributes = context.params[:message_attributes]
validate_single_message(body, attributes, response)
system_attributes = context.params[:message_system_attributes]
validate_single_message(body, attributes, system_attributes, response)
end

def validate_send_message_batch(context, response)
context.params[:entries].each do |entry|
id = entry[:id]
body = entry[:message_body]
attributes = entry[:message_attributes]
system_attributes = entry[:message_system_attributes]
message_response = response.successful.select { |r| r.id == id }[0]
unless message_response.nil?
validate_single_message(body, attributes, message_response)
validate_single_message(body, attributes, system_attributes, message_response)
end
end
end

def validate_single_message(body, attributes, response)
def validate_single_message(body, attributes, system_attributes, response)
validate_body(body, response)
unless attributes.nil? || attributes.empty?
validate_attributes(attributes, response)
end
unless system_attributes.nil? || system_attributes.empty?
validate_system_attributes(system_attributes, response)
end
end

def validate_body(body, response)
Expand Down Expand Up @@ -83,10 +88,27 @@ def validate_attributes(attributes, response)
end
end

def validate_system_attributes(system_attributes, response)
calculated_md5 = md5_of_message_system_attributes(system_attributes)
returned_md5 = response.md5_of_message_system_attributes
if returned_md5 != calculated_md5
error_message = mismatch_error_message(
mullermp marked this conversation as resolved.
Show resolved Hide resolved
'message system attributes',
calculated_md5,
returned_md5,
response)
mullermp marked this conversation as resolved.
Show resolved Hide resolved
raise Aws::Errors::ChecksumError, error_message
end
end

def md5_of_message_body(message_body)
OpenSSL::Digest::MD5.hexdigest(message_body)
end

# MD5 of Message Attributes and System Attributes are effectively
# the same calculation. However, keeping these as two methods because
# they are modeled as two different shapes.
###
jterapin marked this conversation as resolved.
Show resolved Hide resolved
def md5_of_message_attributes(message_attributes)
encoded = { }
message_attributes.each do |name, attribute|
Expand All @@ -110,6 +132,30 @@ def md5_of_message_attributes(message_attributes)
OpenSSL::Digest::MD5.hexdigest(buffer)
end

def md5_of_message_system_attributes(message_system_attributes)
encoded = { }
message_system_attributes.each do |name, attribute|
name = name.to_s
encoded[name] = String.new
data_type_without_label = DATA_TYPE.match(attribute[:data_type])[1]
encoded[name] << encode_length_and_bytes(name) <<
encode_length_and_bytes(attribute[:data_type]) <<
[TRANSPORT_TYPE_ENCODINGS[data_type_without_label]].pack('C'.freeze)
mullermp marked this conversation as resolved.
Show resolved Hide resolved

if attribute[:string_value] != nil
mullermp marked this conversation as resolved.
Show resolved Hide resolved
encoded[name] << encode_length_and_string(attribute[:string_value])
elsif attribute[:binary_value] != nil
encoded[name] << encode_length_and_bytes(attribute[:binary_value])
end
end

buffer = encoded.keys.sort.reduce(String.new) do |string, name|
string << encoded[name]
end
OpenSSL::Digest::MD5.hexdigest(buffer)
end
###

def encode_length_and_string(string)
string = String.new(string)
string.encode!(NORMALIZED_ENCODING)
Expand Down
46 changes: 41 additions & 5 deletions gems/aws-sdk-sqs/spec/client/verify_checksums_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,25 @@ class Client
}
}}

let(:message_system_attributes) {{
'ccc' => {
string_value: 'test',
data_type: 'String'
},
aaa: {
binary_value: [ 2, 3, 4 ].pack('C*'),
data_type: 'Binary'
},
zzz: {
data_type: 'Number',
string_value: '0230.01'
},
'öther_encodings' => {
data_type: 'String',
string_value: 'Tüst'.encode('ISO-8859-1')
}
}}

it 'is enabled by default' do
client = Client.new(
region: 'us-east-1',
Expand All @@ -52,13 +71,15 @@ class Client

describe '#send_message' do
let(:md5_of_attributes) { '756d7f4338696745d063b420a2f7e502' }
let(:md5_of_system_attributes) { '756d7f4338696745d063b420a2f7e502' }

before(:each) do
response_body = <<-JSON
{
"MD5OfMessageAttributes": "#{md5_of_attributes}",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MessageId": "5fea7756-0ea4-451a-a703-a558b933e274"
"MessageId": "5fea7756-0ea4-451a-a703-a558b933e274",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MD5OfMessageAttributes": "#{md5_of_attributes}",
"MD5OfMessageSystemAttributes": "#{md5_of_system_attributes}"
}
JSON

Expand All @@ -77,6 +98,7 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.not_to raise_error
end
Expand All @@ -87,17 +109,22 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: {},
message_system_attributes: {}
)
}.not_to raise_error
end

context 'when data types have custom labels' do
let(:md5_of_attributes) { '5b7ef6c8a8d46001c7cdadaeea917aa4' }
let(:md5_of_system_attributes) { '5b7ef6c8a8d46001c7cdadaeea917aa4' }

before(:each) do
message_attributes.keys.each do |attribute_name|
message_attributes[attribute_name][:data_type] += '.test'
end
message_system_attributes.keys.each do |attribute_name|
message_system_attributes[attribute_name][:data_type] += '.test'
end
end

it 'does not raise an error if checksums match' do
Expand All @@ -106,6 +133,7 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.not_to raise_error
end
Expand All @@ -117,17 +145,20 @@ class Client
queue_url:'https://queue.url',
message_body: message_body + 'junk',
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::Errors::ChecksumError)
end

it 'raises when the md5 checksums do not match for the body' do
message_attributes['ccc'][:string_value] += 'junk'
message_system_attributes['ccc'][:string_value] += 'junk'
expect {
client.send_message(
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::Errors::ChecksumError)
end
Expand All @@ -146,14 +177,14 @@ class Client
queue_url:'https://queue.url',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
)
}.to raise_error(Aws::SQS::Errors::ServiceError)
end

end

describe '#send_message_batch' do

before(:each) do
client.handle(step: :send) do |context|
context.http_response.signal_done(
Expand All @@ -165,7 +196,8 @@ class Client
{
"Id": "msg-id",
"MD5OfMessageBody": "900150983cd24fb0d6963f7d28e17f72",
"MD5OfMessageAttributes": "756d7f4338696745d063b420a2f7e502"
"MD5OfMessageAttributes": "756d7f4338696745d063b420a2f7e502",
"MD5OfMessageSystemAttributes": "756d7f4338696745d063b420a2f7e502"
}
]
}
Expand All @@ -183,6 +215,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand All @@ -198,6 +231,7 @@ class Client
id: 'msg-id',
message_body: message_body + 'junk',
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand All @@ -214,6 +248,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand Down Expand Up @@ -245,6 +280,7 @@ class Client
id: 'msg-id',
message_body: message_body,
message_attributes: message_attributes,
message_system_attributes: message_system_attributes
}
]
)
Expand Down