From 4a853c82fe43dbe2328fa648a89a0f25f0d8f073 Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Mon, 7 Nov 2022 19:46:54 -0500 Subject: [PATCH 1/5] tests: extract streaming tests We'll move these to test_helper.rb soon, so we can test both HTTP and HTTPS --- tests/basic_tests.rb | 111 ++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/tests/basic_tests.rb b/tests/basic_tests.rb index a800862c..390e398e 100644 --- a/tests/basic_tests.rb +++ b/tests/basic_tests.rb @@ -23,70 +23,73 @@ end end -Shindo.tests('Excon streaming basics') do - pending if RUBY_PLATFORM == 'java' # need to find suitable server for jruby - with_unicorn('streaming.ru') do - # expected values: the response, in pieces, and a timeout after each piece - res = %w{Hello streamy world} - timeout = 0.1 - - # expect the full response as a string - # and expect it to take a (timeout * pieces) seconds - tests('simple blocking request on streaming endpoint').returns([res.join(''),'response time ok']) do - start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/simple').body - - if Time.now - start <= timeout*3 - [ret, 'streaming response came too quickly'] - else - [ret, 'response time ok'] - end +# expected values: the response, in pieces, and a timeout after each piece +STREAMING_PIECES = %w{Hello streamy world} +STREAMING_TIMEOUT = 0.1 + +def streaming_tests + # expect the full response as a string + # and expect it to take a (timeout * pieces) seconds + tests('simple blocking request on streaming endpoint').returns([STREAMING_PIECES.join(''),'response time ok']) do + start = Time.now + ret = Excon.get('http://127.0.0.1:9292/streamed/simple').body + + if Time.now - start <= STREAMING_TIMEOUT*3 + [ret, 'streaming response came too quickly'] + else + [ret, 'response time ok'] end + end - # expect the full response as a string and expect it to - # take a (timeout * pieces) seconds (with fixed Content-Length header) - tests('simple blocking request on streaming endpoint with fixed length').returns([res.join(''),'response time ok']) do - start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/fixed_length').body + # expect the full response as a string and expect it to + # take a (timeout * pieces) seconds (with fixed Content-Length header) + tests('simple blocking request on streaming endpoint with fixed length').returns([STREAMING_PIECES.join(''),'response time ok']) do + start = Time.now + ret = Excon.get('http://127.0.0.1:9292/streamed/fixed_length').body - if Time.now - start <= timeout*3 - [ret, 'streaming response came too quickly'] - else - [ret, 'response time ok'] - end + if Time.now - start <= STREAMING_TIMEOUT*3 + [ret, 'streaming response came too quickly'] + else + [ret, 'response time ok'] end + end - # expect each response piece to arrive to the body right away - # and wait for timeout until next one arrives - def timed_streaming_test(endpoint, timeout) - ret = [] - timing = 'response times ok' - start = Time.now - Excon.get(endpoint, :response_block => lambda do |c,r,t| - # add the response - ret.push(c) - # check if the timing is ok - # each response arrives after timeout and before timeout + 1 - cur_time = Time.now - start - if cur_time < ret.length * timeout or cur_time > (ret.length+1) * timeout - timing = 'response time not ok!' - end - end) - # validate the final timing - if Time.now - start <= timeout*3 - timing = 'final timing was not ok!' + # expect each response piece to arrive to the body right away + # and wait for timeout until next one arrives + def timed_streaming_test(endpoint, timeout) + ret = [] + timing = 'response times ok' + start = Time.now + Excon.get(endpoint, :response_block => lambda do |c,r,t| + # add the response + ret.push(c) + # check if the timing is ok + # each response arrives after timeout and before timeout + 1 + cur_time = Time.now - start + if cur_time < ret.length * timeout or cur_time > (ret.length+1) * timeout + timing = 'response time not ok!' end - [ret, timing] + end) + # validate the final timing + if Time.now - start <= timeout*3 + timing = 'final timing was not ok!' end + [ret, timing] + end - tests('simple request with response_block on streaming endpoint').returns([res,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/simple', timeout) - end + tests('simple request with response_block on streaming endpoint').returns([STREAMING_PIECES,'response times ok']) do + timed_streaming_test('http://127.0.0.1:9292/streamed/simple', STREAMING_TIMEOUT) + end - tests('simple request with response_block on streaming endpoint with fixed length').returns([res,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/fixed_length', timeout) - end + tests('simple request with response_block on streaming endpoint with fixed length').returns([STREAMING_PIECES,'response times ok']) do + timed_streaming_test('http://127.0.0.1:9292/streamed/fixed_length', STREAMING_TIMEOUT) + end +end +Shindo.tests('Excon streaming basics') do + pending if RUBY_PLATFORM == 'java' # need to find suitable server for jruby + with_unicorn('streaming.ru') do + streaming_tests end end From 8beb160705360caa19a405dc48e05a4dee6ce14e Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Mon, 7 Nov 2022 19:48:16 -0500 Subject: [PATCH 2/5] tests: Move streaming tests to test_helper --- tests/basic_tests.rb | 63 ------------------------------------------- tests/test_helper.rb | 64 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/tests/basic_tests.rb b/tests/basic_tests.rb index 390e398e..780e45c4 100644 --- a/tests/basic_tests.rb +++ b/tests/basic_tests.rb @@ -23,69 +23,6 @@ end end -# expected values: the response, in pieces, and a timeout after each piece -STREAMING_PIECES = %w{Hello streamy world} -STREAMING_TIMEOUT = 0.1 - -def streaming_tests - # expect the full response as a string - # and expect it to take a (timeout * pieces) seconds - tests('simple blocking request on streaming endpoint').returns([STREAMING_PIECES.join(''),'response time ok']) do - start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/simple').body - - if Time.now - start <= STREAMING_TIMEOUT*3 - [ret, 'streaming response came too quickly'] - else - [ret, 'response time ok'] - end - end - - # expect the full response as a string and expect it to - # take a (timeout * pieces) seconds (with fixed Content-Length header) - tests('simple blocking request on streaming endpoint with fixed length').returns([STREAMING_PIECES.join(''),'response time ok']) do - start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/fixed_length').body - - if Time.now - start <= STREAMING_TIMEOUT*3 - [ret, 'streaming response came too quickly'] - else - [ret, 'response time ok'] - end - end - - # expect each response piece to arrive to the body right away - # and wait for timeout until next one arrives - def timed_streaming_test(endpoint, timeout) - ret = [] - timing = 'response times ok' - start = Time.now - Excon.get(endpoint, :response_block => lambda do |c,r,t| - # add the response - ret.push(c) - # check if the timing is ok - # each response arrives after timeout and before timeout + 1 - cur_time = Time.now - start - if cur_time < ret.length * timeout or cur_time > (ret.length+1) * timeout - timing = 'response time not ok!' - end - end) - # validate the final timing - if Time.now - start <= timeout*3 - timing = 'final timing was not ok!' - end - [ret, timing] - end - - tests('simple request with response_block on streaming endpoint').returns([STREAMING_PIECES,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/simple', STREAMING_TIMEOUT) - end - - tests('simple request with response_block on streaming endpoint with fixed length').returns([STREAMING_PIECES,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/fixed_length', STREAMING_TIMEOUT) - end -end - Shindo.tests('Excon streaming basics') do pending if RUBY_PLATFORM == 'java' # need to find suitable server for jruby with_unicorn('streaming.ru') do diff --git a/tests/test_helper.rb b/tests/test_helper.rb index 5bc68dc4..70702556 100644 --- a/tests/test_helper.rb +++ b/tests/test_helper.rb @@ -213,6 +213,70 @@ def basic_tests(url = 'http://127.0.0.1:9292', options = {}) end +# expected values: the response, in pieces, and a timeout after each piece +STREAMING_PIECES = %w{Hello streamy world} +STREAMING_TIMEOUT = 0.1 + +def streaming_tests + # expect the full response as a string + # and expect it to take a (timeout * pieces) seconds + tests('simple blocking request on streaming endpoint').returns([STREAMING_PIECES.join(''),'response time ok']) do + start = Time.now + ret = Excon.get('http://127.0.0.1:9292/streamed/simple').body + + if Time.now - start <= STREAMING_TIMEOUT*3 + [ret, 'streaming response came too quickly'] + else + [ret, 'response time ok'] + end + end + + # expect the full response as a string and expect it to + # take a (timeout * pieces) seconds (with fixed Content-Length header) + tests('simple blocking request on streaming endpoint with fixed length').returns([STREAMING_PIECES.join(''),'response time ok']) do + start = Time.now + ret = Excon.get('http://127.0.0.1:9292/streamed/fixed_length').body + + if Time.now - start <= STREAMING_TIMEOUT*3 + [ret, 'streaming response came too quickly'] + else + [ret, 'response time ok'] + end + end + + # expect each response piece to arrive to the body right away + # and wait for timeout until next one arrives + def timed_streaming_test(endpoint, timeout) + ret = [] + timing = 'response times ok' + start = Time.now + Excon.get(endpoint, :response_block => lambda do |c,r,t| + # add the response + ret.push(c) + # check if the timing is ok + # each response arrives after timeout and before timeout + 1 + cur_time = Time.now - start + if cur_time < ret.length * timeout or cur_time > (ret.length+1) * timeout + timing = 'response time not ok!' + end + end) + # validate the final timing + if Time.now - start <= timeout*3 + timing = 'final timing was not ok!' + end + [ret, timing] + end + + tests('simple request with response_block on streaming endpoint').returns([STREAMING_PIECES,'response times ok']) do + timed_streaming_test('http://127.0.0.1:9292/streamed/simple', STREAMING_TIMEOUT) + end + + tests('simple request with response_block on streaming endpoint with fixed length').returns([STREAMING_PIECES,'response times ok']) do + timed_streaming_test('http://127.0.0.1:9292/streamed/fixed_length', STREAMING_TIMEOUT) + end +end + + PROXY_ENV_VARIABLES = %w{http_proxy https_proxy no_proxy} # All lower-case def env_init(env={}) From bd10b63694ac0f27fd8ece687b13dd597f540c4f Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Mon, 7 Nov 2022 20:05:10 -0500 Subject: [PATCH 3/5] tests: Run streaming tests in SSL mode Notice a couple of tests fail now --- tests/basic_tests.rb | 13 +++++++-- tests/test_helper.rb | 69 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/tests/basic_tests.rb b/tests/basic_tests.rb index 780e45c4..95f7c60b 100644 --- a/tests/basic_tests.rb +++ b/tests/basic_tests.rb @@ -24,9 +24,16 @@ end Shindo.tests('Excon streaming basics') do - pending if RUBY_PLATFORM == 'java' # need to find suitable server for jruby - with_unicorn('streaming.ru') do - streaming_tests + tests('http') do + pending if RUBY_PLATFORM == 'java' # need to find suitable server for jruby + with_unicorn('streaming.ru') do + streaming_tests('http') + end + end + tests('https') do + with_ssl_streaming(9292, STREAMING_PIECES, STREAMING_TIMEOUT) do + streaming_tests('https') + end end end diff --git a/tests/test_helper.rb b/tests/test_helper.rb index 70702556..e355d733 100644 --- a/tests/test_helper.rb +++ b/tests/test_helper.rb @@ -3,6 +3,7 @@ require 'excon' require 'delorean' require 'open4' +require 'webrick' require './spec/helpers/warning_helpers.rb' @@ -217,12 +218,18 @@ def basic_tests(url = 'http://127.0.0.1:9292', options = {}) STREAMING_PIECES = %w{Hello streamy world} STREAMING_TIMEOUT = 0.1 -def streaming_tests +def streaming_tests(protocol) + conn = nil + test do + conn = Excon.new("#{protocol}://127.0.0.1:9292/", :ssl_verify_peer => false) + true + end + # expect the full response as a string # and expect it to take a (timeout * pieces) seconds tests('simple blocking request on streaming endpoint').returns([STREAMING_PIECES.join(''),'response time ok']) do start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/simple').body + ret = conn.request(:method => :get, :path => '/streamed/simple').body if Time.now - start <= STREAMING_TIMEOUT*3 [ret, 'streaming response came too quickly'] @@ -235,7 +242,7 @@ def streaming_tests # take a (timeout * pieces) seconds (with fixed Content-Length header) tests('simple blocking request on streaming endpoint with fixed length').returns([STREAMING_PIECES.join(''),'response time ok']) do start = Time.now - ret = Excon.get('http://127.0.0.1:9292/streamed/fixed_length').body + ret = conn.request(:method => :get, :path => '/streamed/fixed_length').body if Time.now - start <= STREAMING_TIMEOUT*3 [ret, 'streaming response came too quickly'] @@ -246,11 +253,11 @@ def streaming_tests # expect each response piece to arrive to the body right away # and wait for timeout until next one arrives - def timed_streaming_test(endpoint, timeout) + def timed_streaming_test(conn, path, timeout) ret = [] timing = 'response times ok' start = Time.now - Excon.get(endpoint, :response_block => lambda do |c,r,t| + conn.request(:method => :get, :path => path, :response_block => lambda do |c,r,t| # add the response ret.push(c) # check if the timing is ok @@ -268,11 +275,11 @@ def timed_streaming_test(endpoint, timeout) end tests('simple request with response_block on streaming endpoint').returns([STREAMING_PIECES,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/simple', STREAMING_TIMEOUT) + timed_streaming_test(conn, '/streamed/simple', STREAMING_TIMEOUT) end tests('simple request with response_block on streaming endpoint with fixed length').returns([STREAMING_PIECES,'response times ok']) do - timed_streaming_test('http://127.0.0.1:9292/streamed/fixed_length', STREAMING_TIMEOUT) + timed_streaming_test(conn, '/streamed/fixed_length', STREAMING_TIMEOUT) end end @@ -393,3 +400,51 @@ def with_server(name) ensure cleanup_process(pid) end + +# A tiny fake SSL streaming server +def with_ssl_streaming(port, pieces, delay) + key_file = File.join(File.dirname(__FILE__), 'data', '127.0.0.1.cert.key') + cert_file = File.join(File.dirname(__FILE__), 'data', '127.0.0.1.cert.crt') + + ctx = OpenSSL::SSL::SSLContext.new + ctx.key = OpenSSL::PKey::RSA.new(File.read(key_file)) + ctx.cert = OpenSSL::X509::Certificate.new(File.read(cert_file)) + + tcp = TCPServer.new(port) + ssl = OpenSSL::SSL::SSLServer.new(tcp, ctx) + + Thread.new do + loop do + begin + conn = ssl.accept + rescue IOError => e + # we're closing the socket from another thread, which makes `accept` complain + break if /stream closed/ =~ e.to_s + raise + end + + Thread.new do + begin + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(conn) + + conn << "HTTP/1.1 200 OK\r\n\r\n" + if req.path == "streamed/fixed_length" + conn << "Content-Length: #{pieces.join.length}\r\n" + end + conn.flush + + pieces.each do |piece| + sleep(delay) + conn.write(piece) + conn.flush + end + ensure + conn.close + end + end + end + end + yield + ssl.close +end From e6aabd055e44ed05236b638529ffa30850f27602 Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Mon, 7 Nov 2022 20:06:02 -0500 Subject: [PATCH 4/5] socket: Add guard around select_with_timeout in SSL mode Allows SSL streaming tests to pass. Without this, we wait even though we already have some data to return. --- lib/excon/socket.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/excon/socket.rb b/lib/excon/socket.rb index 702a2956..23e42d33 100644 --- a/lib/excon/socket.rb +++ b/lib/excon/socket.rb @@ -185,7 +185,9 @@ def read_nonblock(max_length) end rescue OpenSSL::SSL::SSLError => error if error.message == 'read would block' - select_with_timeout(@socket, :read) && retry + if @read_buffer.empty? + select_with_timeout(@socket, :read) && retry + end else raise(error) end From 464b7d5bfb113f402f92a1011d14bf76d1cb825b Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Mon, 7 Nov 2022 20:06:50 -0500 Subject: [PATCH 5/5] socket: Remove guard in blocking mode This has no effect, since @read_buffer is completely unused in blocking mode. It just keeps the code cleaner. --- lib/excon/socket.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/excon/socket.rb b/lib/excon/socket.rb index 23e42d33..c5bb58fe 100644 --- a/lib/excon/socket.rb +++ b/lib/excon/socket.rb @@ -221,9 +221,7 @@ def read_block(max_length) raise(error) end rescue *READ_RETRY_EXCEPTION_CLASSES - if @read_buffer.empty? - select_with_timeout(@socket, :read) && retry - end + select_with_timeout(@socket, :read) && retry rescue EOFError @eof = true end