From efd06ca941f44222dcddba7be0cf1edea74ce01e Mon Sep 17 00:00:00 2001 From: Alexander Clouter Date: Mon, 13 Nov 2017 23:29:20 +0000 Subject: [PATCH] transparent (content-encoding) compression --- include/hackney_lib.hrl | 4 +++- src/hackney.erl | 6 ++++++ src/hackney_http.erl | 40 +++++++++++++++++++++++++++++++++------- src/hackney_request.erl | 25 +++++++++++++++---------- 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/include/hackney_lib.hrl b/include/hackney_lib.hrl index 28a7320d..da1d8584 100644 --- a/include/hackney_lib.hrl +++ b/include/hackney_lib.hrl @@ -26,8 +26,10 @@ partial_headers = [] :: list(), clen = undefined :: integer() | undefined | bad_int, te = <<>> :: binary(), + ce = <<>> :: binary(), connection = <<>> :: binary(), ctype = <<>> :: binary(), location = <<>> :: binary(), - body_state = waiting :: atom() | tuple() + body_state = waiting :: atom() | tuple(), + encoding :: {atom(), any()} | undefined }). diff --git a/src/hackney.erl b/src/hackney.erl index 84c3626b..a0227986 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -94,6 +94,7 @@ cancel_request(Ref) -> %% @doc set client options. %% Options are: +%% - `compress': request compression and transparently decompress %% - `async': to fetch the response asynchronously %% - `{async, once}': to receive the response asynchronously once time. %% To receive the next message use the function `hackney:stream_next/1'. @@ -211,6 +212,8 @@ request(Method, URL, Headers, Body) -> %% directly. The response is `{ok, Status, Headers, Body}' %%
  • `max_body': sets maximum allowed size of the body if %% with_body is true
    • +%%
  • `compress': request that the server sends the body compressed +%% and instructs hackney to transparently decompress it
    • %%
  • `async': receive the response asynchronously %% The function return {ok, StreamRef}. %% When {async, once} is used the response will be received only once. To @@ -985,6 +988,9 @@ maybe_update_req(State) -> parse_options([], State) -> State; +parse_options([compress | Rest], State = #client{headers=Headers}) -> + Headers2 = hackney_headers:store(<<"accept-encoding">>, <<"gzip, deflate">>, Headers), + parse_options(Rest, State#client{headers=Headers2}); parse_options([async | Rest], State) -> parse_options(Rest, State#client{async=true}); parse_options([{async, Async} | Rest], State) -> diff --git a/src/hackney_http.erl b/src/hackney_http.erl index 347fdf8b..5c2f9e45 100644 --- a/src/hackney_http.erl +++ b/src/hackney_http.erl @@ -305,6 +305,9 @@ parse_header(Line, St) -> <<"transfer-encoding">> -> TE = hackney_bstr:to_lower(hackney_bstr:trim(Value)), St#hparser{te=TE}; + <<"content-encoding">> -> + CE = hackney_bstr:to_lower(hackney_bstr:trim(Value)), + St#hparser{ce=CE}; <<"connection">> -> Connection = hackney_bstr:to_lower(hackney_bstr:trim(Value)), St#hparser{connection=Connection}; @@ -330,9 +333,31 @@ parse_trailers(St, Acc) -> _ -> error end. -parse_body(#hparser{body_state=waiting, method= <<"HEAD">>, buffer=Buffer}) -> - {done, Buffer}; -parse_body(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) -> +parse_body(#hparser{body_state=waiting, method=Method, buffer=Buffer, clen=Length}) when Method == <<"HEAD">>; Length =:= 0 -> + {done, Buffer}; +parse_body(St=#hparser{body_state=waiting, ce=CE}) when CE == <<"gzip">>; CE == <<"deflate">> -> + MaxWBITS = 15, % zconf.h + WB = MaxWBITS + if CE == <<"gzip">> -> 16; true -> 0 end, % http://www.zlib.net/manual.html#Advanced + Z = zlib:open(), + ok = zlib:inflateInit(Z, WB), + parse_body2(St#hparser{encoding={zlib,Z}}); +parse_body(St=#hparser{encoding={zlib,Z}}) -> + case parse_body2(St) of + {ok, Chunk, St2} -> + Chunk2 = iolist_to_binary(zlib:inflate(Z, Chunk)), + {ok, Chunk2, St2}; + {done, Rest} -> + Rest2 = iolist_to_binary(zlib:inflate(Z, Rest)), + ok = zlib:inflateEnd(Z), + ok = zlib:close(Z), + {done, Rest2}; + Else -> + Else + end; +parse_body(St) -> + parse_body2(St). + +parse_body2(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) -> case {TE, Length} of {<<"chunked">>, _} -> parse_body(St#hparser{body_state= @@ -346,14 +371,13 @@ parse_body(St=#hparser{body_state=waiting, te=TE, clen=Length, buffer=Buffer}) - St#hparser{body_state={stream, fun te_identity/2, {0, Length}, fun ce_identity/1}} ) end; -parse_body(#hparser{body_state=done, buffer=Buffer}) -> +parse_body2(#hparser{body_state=done, buffer=Buffer}) -> {done, Buffer}; -parse_body(St=#hparser{buffer=Buffer, body_state={stream, _, _, _}}) when byte_size(Buffer) > 0 -> +parse_body2(St=#hparser{buffer=Buffer, body_state={stream, _, _, _}}) when byte_size(Buffer) > 0 -> transfer_decode(Buffer, St#hparser{buffer= <<>>}); -parse_body(St) -> +parse_body2(St) -> {more, St, <<>>}. - -spec transfer_decode(binary(), #hparser{}) -> {ok, binary(), #hparser{}} | {done, binary()} | {error, atom()}. transfer_decode(Data, St=#hparser{ @@ -518,6 +542,8 @@ get_property(method, #hparser{method=Method}) -> Method; get_property(transfer_encoding, #hparser{te=TE}) -> TE; +get_property(content_encoding, #hparser{ce=CE}) -> + CE; get_property(content_length, #hparser{clen=CLen}) -> CLen; get_property(connection, #hparser{connection=Connection}) -> diff --git a/src/hackney_request.erl b/src/hackney_request.erl index c399d29a..30b05feb 100644 --- a/src/hackney_request.erl +++ b/src/hackney_request.erl @@ -57,30 +57,35 @@ perform(Client0, {Method0, Path, Headers0, Body0}) -> %% add host eventually Headers2 = maybe_add_host(Headers1, Client0#client.netloc), + Compress = proplists:get_value(compress, Options, false), + Headers3 = if Compress -> + hackney_headers_new:store(<<"accept-encoding">>, <<"gzip, deflate">>, Headers2); + true -> Headers2 end, + %% get expect headers - Expect = expectation(Headers2), + Expect = expectation(Headers3), %% build headers with the body. {FinalHeaders, ReqType, Body, Client1} = case Body0 of stream -> - {Headers2, ReqType0, stream, Client0}; + {Headers3, ReqType0, stream, Client0}; stream_multipart -> - handle_multipart_body(Headers2, ReqType0, Client0); + handle_multipart_body(Headers3, ReqType0, Client0); {stream_multipart, Size} -> - handle_multipart_body(Headers2, ReqType0, Size, Client0); + handle_multipart_body(Headers3, ReqType0, Size, Client0); {stream_multipart, Size, Boundary} -> - handle_multipart_body(Headers2, ReqType0, + handle_multipart_body(Headers3, ReqType0, Size, Boundary, Client0); <<>> when Method =:= <<"POST">> orelse Method =:= <<"PUT">> -> - handle_body(Headers2, ReqType0, Body0, Client0); + handle_body(Headers3, ReqType0, Body0, Client0); [] when Method =:= <<"POST">> orelse Method =:= <<"PUT">> -> - handle_body(Headers2, ReqType0, Body0, Client0); + handle_body(Headers3, ReqType0, Body0, Client0); <<>> -> - {Headers2, ReqType0, Body0, Client0}; + {Headers3, ReqType0, Body0, Client0}; [] -> - {Headers2, ReqType0, Body0, Client0}; + {Headers3, ReqType0, Body0, Client0}; _ -> - handle_body(Headers2, ReqType0, Body0, Client0) + handle_body(Headers3, ReqType0, Body0, Client0) end, %% build final client record