Skip to content

Commit

Permalink
transparent (content-encoding) compression
Browse files Browse the repository at this point in the history
  • Loading branch information
jimdigriz committed Nov 24, 2019
1 parent 5b7363c commit a793e7b
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 18 deletions.
4 changes: 3 additions & 1 deletion include/hackney_lib.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
partial_headers = [] :: list(),
clen = undefined :: integer() | undefined | bad_int,
te = <<>> :: binary(),
ce = <<>> :: binary(),
connection = <<>> :: binary(),
ctype = <<>> :: binary(),
location = <<>> :: binary(),
body_state = waiting :: atom() | tuple()
body_state = waiting :: atom() | tuple(),
encoding :: {atom(), any()} | undefined
}).
6 changes: 6 additions & 0 deletions src/hackney.erl
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ cancel_request(Ref) ->

%% @doc set client options.
%% Options are:
%% - `compress': request compression and transparently decompress
%% - `async': to fetch the response asynchronously
%% - `{async, once}': to receive the response asynchronously one time.
%% To receive the next message use the function `hackney:stream_next/1'.
Expand Down Expand Up @@ -211,6 +212,8 @@ request(Method, URL, Headers, Body) ->
%% directly. The response is `{ok, Status, Headers, Body}'</li>
%% <li>`max_body': sets maximum allowed size of the body if
%% with_body is true</li>
%% <li>`compress': request that the server sends the body compressed
%% and instructs hackney to transparently decompress it</li>
%% <li>`async': receive the response asynchronously
%% The function return {ok, StreamRef}.
%% When {async, once} is used the response will be received only once. To
Expand Down Expand Up @@ -988,6 +991,9 @@ maybe_update_req(State) ->

parse_options([], State) ->
State;
parse_options([compress | Rest], State = #client{headers=Headers}) ->
Headers2 = hackney_headers:store(<<"accept-encoding">>, <<"gzip, deflate">>, Headers),
parse_options(Rest, State#client{headers=Headers2});
parse_options([async | Rest], State) ->
parse_options(Rest, State#client{async=true});
parse_options([{async, Async} | Rest], State) ->
Expand Down
40 changes: 33 additions & 7 deletions src/hackney_http.erl
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,9 @@ parse_header(Line, St) ->
<<"transfer-encoding">> ->
TE = hackney_bstr:to_lower(hackney_bstr:trim(Value)),
St#hparser{te=TE};
<<"content-encoding">> ->
CE = hackney_bstr:to_lower(hackney_bstr:trim(Value)),
St#hparser{ce=CE};
<<"connection">> ->
Connection = hackney_bstr:to_lower(hackney_bstr:trim(Value)),
St#hparser{connection=Connection};
Expand All @@ -330,9 +333,31 @@ parse_trailers(St, Acc) ->
_ -> error
end.

parse_body(#hparser{body_state=waiting, method= <<"HEAD">>, buffer=Buffer}) ->
{done, Buffer};
parse_body(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) ->
parse_body(#hparser{body_state=waiting, method=Method, buffer=Buffer, clen=Length}) when Method == <<"HEAD">>; Length =:= 0 ->
{done, Buffer};
parse_body(St=#hparser{body_state=waiting, ce=CE}) when CE == <<"gzip">>; CE == <<"deflate">> ->
MaxWBITS = 15, % zconf.h
WB = MaxWBITS + if CE == <<"gzip">> -> 16; true -> 0 end, % http://www.zlib.net/manual.html#Advanced
Z = zlib:open(),
ok = zlib:inflateInit(Z, WB),
parse_body2(St#hparser{encoding={zlib,Z}});
parse_body(St=#hparser{encoding={zlib,Z}}) ->
case parse_body2(St) of
{ok, Chunk, St2} ->
Chunk2 = iolist_to_binary(zlib:inflate(Z, Chunk)),
{ok, Chunk2, St2};
{done, Rest} ->
Rest2 = iolist_to_binary(zlib:inflate(Z, Rest)),
ok = zlib:inflateEnd(Z),
ok = zlib:close(Z),
{done, Rest2};
Else ->
Else
end;
parse_body(St) ->
parse_body2(St).

parse_body2(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) ->
case {TE, Length} of
{<<"chunked">>, _} ->
parse_body(St#hparser{body_state=
Expand All @@ -346,14 +371,13 @@ parse_body(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) -
St#hparser{body_state={stream, fun te_identity/2, {0, Length}, fun ce_identity/1}}
)
end;
parse_body(#hparser{body_state=done, buffer=Buffer}) ->
parse_body2(#hparser{body_state=done, buffer=Buffer}) ->
{done, Buffer};
parse_body(St=#hparser{buffer=Buffer, body_state={stream, _, _, _}}) when byte_size(Buffer) > 0 ->
parse_body2(St=#hparser{buffer=Buffer, body_state={stream, _, _, _}}) when byte_size(Buffer) > 0 ->
transfer_decode(Buffer, St#hparser{buffer= <<>>});
parse_body(St) ->
parse_body2(St) ->
{more, St, <<>>}.


-spec transfer_decode(binary(), #hparser{})
-> {ok, binary(), #hparser{}} | {done, binary()} | {error, atom()}.
transfer_decode(Data, St=#hparser{
Expand Down Expand Up @@ -518,6 +542,8 @@ get_property(method, #hparser{method=Method}) ->
Method;
get_property(transfer_encoding, #hparser{te=TE}) ->
TE;
get_property(content_encoding, #hparser{ce=CE}) ->
CE;
get_property(content_length, #hparser{clen=CLen}) ->
CLen;
get_property(connection, #hparser{connection=Connection}) ->
Expand Down
25 changes: 15 additions & 10 deletions src/hackney_request.erl
Original file line number Diff line number Diff line change
Expand Up @@ -57,30 +57,35 @@ perform(Client0, {Method0, Path, Headers0, Body0}) ->
%% add host eventually
Headers2 = maybe_add_host(Headers1, Client0#client.netloc),

Compress = proplists:get_value(compress, Options, false),
Headers3 = if Compress ->
hackney_headers_new:store(<<"accept-encoding">>, <<"gzip, deflate">>, Headers2);
true -> Headers2 end,

%% get expect headers
Expect = expectation(Headers2),
Expect = expectation(Headers3),

%% build headers with the body.
{FinalHeaders, ReqType, Body, Client1} = case Body0 of
stream ->
{Headers2, ReqType0, stream, Client0};
{Headers3, ReqType0, stream, Client0};
stream_multipart ->
handle_multipart_body(Headers2, ReqType0, Client0);
handle_multipart_body(Headers3, ReqType0, Client0);
{stream_multipart, Size} ->
handle_multipart_body(Headers2, ReqType0, Size, Client0);
handle_multipart_body(Headers3, ReqType0, Size, Client0);
{stream_multipart, Size, Boundary} ->
handle_multipart_body(Headers2, ReqType0,
handle_multipart_body(Headers3, ReqType0,
Size, Boundary, Client0);
<<>> when Method =:= <<"POST">> orelse Method =:= <<"PUT">> ->
handle_body(Headers2, ReqType0, Body0, Client0);
handle_body(Headers3, ReqType0, Body0, Client0);
[] when Method =:= <<"POST">> orelse Method =:= <<"PUT">> ->
handle_body(Headers2, ReqType0, Body0, Client0);
handle_body(Headers3, ReqType0, Body0, Client0);
<<>> ->
{Headers2, ReqType0, Body0, Client0};
{Headers3, ReqType0, Body0, Client0};
[] ->
{Headers2, ReqType0, Body0, Client0};
{Headers3, ReqType0, Body0, Client0};
_ ->
handle_body(Headers2, ReqType0, Body0, Client0)
handle_body(Headers3, ReqType0, Body0, Client0)
end,

%% build final client record
Expand Down

0 comments on commit a793e7b

Please sign in to comment.