Skip to content

Commit

Permalink
transparent (content-encoding) compression support
Browse files Browse the repository at this point in the history
  • Loading branch information
jimdigriz committed Oct 3, 2020
1 parent 039db60 commit 7b21e91
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 15 deletions.
1 change: 1 addition & 0 deletions include/hackney.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
partial_headers = [],
version,
clen = nil,
ce = nil,
te = nil,
connection = nil,
method = nil,
Expand Down
3 changes: 3 additions & 0 deletions src/hackney.erl
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ cancel_request(Ref) ->

%% @doc set client options.
%% Options are:
%% - `compress': request compression and transparently decompress
%% - `async': to fetch the response asynchronously
%% - `{async, once}': to receive the response asynchronously one time.
%% To receive the next message use the function `hackney:stream_next/1'.
Expand Down Expand Up @@ -211,6 +212,8 @@ request(Method, URL, Headers, Body) ->
%% directly. The response is `{ok, Status, Headers, Body}'</li>
%% <li>`max_body': sets maximum allowed size of the body if
%% with_body is true</li>
%% <li>`compress': request that the server sends the body compressed
%% and instructs hackney to transparently decompress it</li>
%% <li>`async': receive the response asynchronously
%% The function return {ok, StreamRef}.
%% When {async, once} is used the response will be received only once. To
Expand Down
26 changes: 16 additions & 10 deletions src/hackney_request.erl
Original file line number Diff line number Diff line change
Expand Up @@ -61,30 +61,36 @@ perform(Client0, {Method0, Path0, Headers0, Body0}) ->
%% add host eventually
Headers2 = maybe_add_host(Headers1, Client0#client.netloc),

%% overwrite always as we control Accept-Encoding when 'compress' is set
Headers3 = case proplists:get_value(compress, Options, false) of
true -> hackney_headers_new:store(<<"Accept-Encoding">>, <<"gzip, deflate">>, Headers2);
false -> Headers2
end,

%% get expect headers
Expect = expectation(Headers2),
Expect = expectation(Headers3),

%% build headers with the body.
{FinalHeaders, ReqType, Body, Client1} = case Body0 of
stream ->
{Headers2, ReqType0, stream, Client0};
{Headers3, ReqType0, stream, Client0};
stream_multipart ->
handle_multipart_body(Headers2, ReqType0, Client0);
handle_multipart_body(Headers3, ReqType0, Client0);
{stream_multipart, Size} ->
handle_multipart_body(Headers2, ReqType0, Size, Client0);
handle_multipart_body(Headers3, ReqType0, Size, Client0);
{stream_multipart, Size, Boundary} ->
handle_multipart_body(Headers2, ReqType0,
handle_multipart_body(Headers3, ReqType0,
Size, Boundary, Client0);
<<>> when Method =:= <<"POST">> orelse Method =:= <<"PUT">> ->
handle_body(Headers2, ReqType0, Body0, Client0);
handle_body(Headers3, ReqType0, Body0, Client0);
[] when Method =:= <<"POST">> orelse Method =:= <<"PUT">> ->
handle_body(Headers2, ReqType0, Body0, Client0);
handle_body(Headers3, ReqType0, Body0, Client0);
<<>> ->
{Headers2, ReqType0, Body0, Client0};
{Headers3, ReqType0, Body0, Client0};
[] ->
{Headers2, ReqType0, Body0, Client0};
{Headers3, ReqType0, Body0, Client0};
_ ->
handle_body(Headers2, ReqType0, Body0, Client0)
handle_body(Headers3, ReqType0, Body0, Client0)
end,

%% build final client record
Expand Down
80 changes: 75 additions & 5 deletions src/hackney_response.erl
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,25 @@ wait_headers({headers_complete, Parser}, Client, Status, Headers) ->
[hackney, Client#client.host, response_time],
ResponseTime),
HeadersList = hackney_headers_new:to_list(Headers),
CE = case proplists:get_value(compress, Client#client.options, false) of
true ->
case hackney_headers_new:get_value(<<"content-encoding">>, Headers, nil) of
nil -> nil;
C ->
Z = zlib:open(),
%% inflateInit2 (https://www.zlib.net/manual.html#Advanced)
WindowBits = 15 + if C == <<"gzip">> -> 16; true -> 0 end,
ok = zlib:inflateInit(Z, WindowBits),
ok = case erlang:function_exported(zlib, safeInflate, 2) of
%% OTP-20.0.5 and later
true -> ok;
%% OTP-18.0 and later
false -> zlib:setBufSize(Z, 512 * 1024)
end,
{zlib,Z}
end;
false -> nil
end,
TE = hackney_headers_new:get_value(<<"transfer-encoding">>, Headers, nil),
CLen = case hackney_headers_new:lookup("content-length", Headers) of
[] -> undefined;
Expand All @@ -122,6 +141,7 @@ wait_headers({headers_complete, Parser}, Client, Status, Headers) ->
end,
Client2 = Client#client{parser=Parser,
headers=Headers,
ce=CE,
te=TE,
clen=CLen},
{ok, Status, HeadersList, Client2}.
Expand All @@ -147,17 +167,64 @@ stream_body(Client=#client{parser=Parser, clen=CLen, te=TE}) ->
stream_body(Data, #client{parser=Parser}=Client) ->
stream_body1(hackney_http:execute(Parser, Data), Client).

stream_body1({more, Parser, Buffer}, Client) ->
stream_body1({ok, Data, Parser}, Client = #client{ce={zlib,Z}}) ->
stream_body2(case stream_body_zlib(Z, Data) of
<<>> -> {more, Parser, <<>>};
D when is_binary(D) -> {ok, D, Parser};
E -> {error,E}
end, Client);
stream_body1({done, _Rest}, Client = #client{ce={zlib,_Z}}) ->
stream_body1(done, Client);
stream_body1(done, Client = #client{ce={zlib,Z}, parser=Parser}) ->
stream_body2(case stream_body_zlib(Z, <<>>) of
done -> done;
D when is_binary(D), size(D) > 0 -> {ok, D, Parser};
E -> {error,E}
end, Client);
stream_body1(Result, Client) ->
stream_body2(Result, Client).

stream_body_zlib(Z, Data) ->
case erlang:function_exported(zlib, safeInflate, 2) of
true ->
%% OTP-20.0.5 and later
case zlib:safeInflate(Z, Data) of
{continue, []} when Data == <<>> ->
data_error;
{finished, []} when Data == <<>> ->
case (catch zlib:inflateEnd(Z)) of
ok -> done;
_ -> data_error
end;
{_, Output} ->
iolist_to_binary(Output)
end;
false ->
%% OTP-18.0 and later
case zlib:inflateChunk(Z, Data) of
[] when Data == <<>> ->
case (catch zlib:inflateEnd(Z)) of
ok -> done;
_ -> data_error
end;
{more, Decompressed} ->
iolist_to_binary(Decompressed);
Decompressed ->
iolist_to_binary(Decompressed)
end
end.

stream_body2({more, Parser, Buffer}, Client) ->
stream_body_recv(Buffer, Client#client{parser=Parser});
stream_body1({ok, Data, Parser}, Client) ->
stream_body2({ok, Data, Parser}, Client) ->
{ok, Data, Client#client{parser=Parser}};
stream_body1({done, Rest}, Client) ->
stream_body2({done, Rest}, Client) ->
Client2 = end_stream_body(Rest, Client),
{done, Client2};
stream_body1(done, Client) ->
stream_body2(done, Client) ->
Client2 = end_stream_body(<<>>, Client),
{done, Client2};
stream_body1(Error, _Client) ->
stream_body2(Error, _Client) ->
Error.


Expand Down Expand Up @@ -277,6 +344,9 @@ skip_body(Client) ->
{error, Reason} -> {error, Reason}
end.

end_stream_body(Rest, Client = #client{ce={zlib,Z}}) ->
catch zlib:close(Z),
end_stream_body(Rest, Client#client{ce=nil});
end_stream_body(Rest, Client0) ->
Client = Client0#client{response_state=done,
body_state=done,
Expand Down

0 comments on commit 7b21e91

Please sign in to comment.