diff --git a/.dialyzer_ignore b/.dialyzer_ignore index f0d1175e..adb0d82e 100644 --- a/.dialyzer_ignore +++ b/.dialyzer_ignore @@ -1,5 +1,5 @@ lib/mint/tunnel_proxy.ex:50 -lib/mint/http1.ex:999 +lib/mint/http1.ex:1063 lib/mint/unsafe_proxy.ex:173 lib/mint/unsafe_proxy.ex:198 test/support diff --git a/lib/mint/http.ex b/lib/mint/http.ex index 92b24eb4..d453a15a 100644 --- a/lib/mint/http.ex +++ b/lib/mint/http.ex @@ -844,8 +844,11 @@ defmodule Mint.HTTP do * `{:headers, request_ref, headers}` - returned when the server replied with a list of headers. Headers are in the form `{header_name, header_value}` with `header_name` and `header_value` being strings. A single `:headers` response - will come after the `:status` response. A single `:headers` response may come - after all the `:data` responses if **trailer headers** are present. + will come after the `:status` response and a single `:headers` response may come + after all the `:data` responses if **trailer headers** are present unless + `:stream_headers` is enabled (only available for HTTP/1.1 connections), in which + case any number of `:headers` responses (including none) may come after the + `:status` response and/or after all the `:data` responses. * `{:data, request_ref, binary}` - returned when the server replied with a chunk of response body (as a binary). The request shouldn't be considered done diff --git a/lib/mint/http1.ex b/lib/mint/http1.ex index 87b20a69..369c68e0 100644 --- a/lib/mint/http1.ex +++ b/lib/mint/http1.ex @@ -104,6 +104,7 @@ defmodule Mint.HTTP1 do :case_sensitive_headers, :skip_target_validation, :max_header_list_size, + :stream_headers, requests: :queue.new(), state: :closed, buffer: "", @@ -150,6 +151,11 @@ defmodule Mint.HTTP1 do bytes allowed in a response header section or chunked trailer section. This includes header names, values, and line delimiters. Defaults to 256 KiB. *Available since 1.9.2*. + * `:stream_headers` - (`t:boolean/0`) if set to `true`, response headers and trailer headers + will be emitted as they are parsed, rather than buffered until the complete header section + is received. When enabled, you may receive multiple `{:headers, ref, headers}` responses + for a single request. Defaults to `false`. *Available since v1.10.0*. + """ @spec connect(Types.scheme(), Types.address(), :inet.port_number(), keyword()) :: {:ok, t()} | {:error, Types.error()} @@ -238,6 +244,7 @@ defmodule Mint.HTTP1 do case_sensitive_headers: Keyword.get(opts, :case_sensitive_headers, false), skip_target_validation: Keyword.get(opts, :skip_target_validation, false), max_header_list_size: max_header_list_size, + stream_headers: Keyword.get(opts, :stream_headers, false), optional_responses: validate_optional_response_values(opts) } @@ -744,7 +751,7 @@ defmodule Mint.HTTP1 do end defp decode_headers(conn, request, data, responses, headers) do - case Response.decode_header(data) do + case decode_header(data, conn.stream_headers) do {:ok, {name, value}, rest} -> headers = [{name, value} | headers] @@ -758,7 +765,13 @@ defmodule Mint.HTTP1 do {:ok, :eof, rest} -> case add_header_bytes(conn, request, byte_size(data) - byte_size(rest)) do {:ok, request} -> - responses = [{:headers, request.ref, Enum.reverse(headers)} | responses] + responses = + if conn.stream_headers and headers == [] do + responses + else + [{:headers, request.ref, Enum.reverse(headers)} | responses] + end + request = %{request | state: :body, headers_buffer: [], headers_size: 0} conn = %{conn | buffer: "", request: request} decode(:body, conn, rest, responses) @@ -770,7 +783,19 @@ defmodule Mint.HTTP1 do :more -> case check_header_section_size(conn, request.headers_size + byte_size(data)) do :ok -> - request = %{request | headers_buffer: headers} + {responses, headers_buffer} = + cond do + not conn.stream_headers -> + {responses, headers} + + headers != [] -> + {[{:headers, request.ref, Enum.reverse(headers)} | responses], []} + + true -> + {responses, []} + end + + request = %{request | headers_buffer: headers_buffer} conn = %{conn | buffer: data, request: request} {:ok, conn, responses} @@ -918,7 +943,7 @@ defmodule Mint.HTTP1 do end defp decode_trailer_headers(conn, data, responses, headers) do - case Response.decode_header(data) do + case decode_header(data, conn.stream_headers) do {:ok, {name, value}, rest} -> case add_header_bytes(conn, conn.request, byte_size(data) - byte_size(rest)) do {:ok, request} -> @@ -950,7 +975,24 @@ defmodule Mint.HTTP1 do :more -> case check_header_section_size(conn, conn.request.headers_size + byte_size(data)) do :ok -> - request = %{conn.request | body: {:chunked, :trailer}, headers_buffer: headers} + {responses, headers_buffer} = + cond do + not conn.stream_headers -> + {responses, headers} + + headers != [] -> + responses = + headers + |> Headers.remove_unallowed_trailer() + |> add_trailer_headers(conn.request.ref, responses) + + {responses, []} + + true -> + {responses, []} + end + + request = %{conn.request | body: {:chunked, :trailer}, headers_buffer: headers_buffer} conn = %{conn | buffer: data, request: request} {:ok, conn, responses} @@ -963,6 +1005,28 @@ defmodule Mint.HTTP1 do end end + defp decode_header(data, false = _stream_headers), do: Response.decode_header(data) + + defp decode_header(data, true = _stream_headers) do + # By default, :erlang.decode_packet/3 asks for more data when a packet + # containing a full header ends with a line feed (likely to handle line + # folding). If we get a :more response on a packet that ends with a line + # feed, we append a sentinel byte and attempt to decode again. + with :more <- Response.decode_header(data) do + data_size = byte_size(data) + + case data do + <<_::binary-size(^data_size - 1), ?\n>> -> + with {:ok, {name, value}, <<0>>} <- Response.decode_header(<>) do + {:ok, {name, value}, ""} + end + + _ -> + :more + end + end + end + defp next_request(%{request: nil} = conn, data, responses) do # TODO: Figure out if we should keep buffering even though there are no # requests in flight diff --git a/test/mint/http1/conn_test.exs b/test/mint/http1/conn_test.exs index 4a5dd281..5e6f3948 100644 --- a/test/mint/http1/conn_test.exs +++ b/test/mint/http1/conn_test.exs @@ -1273,6 +1273,170 @@ defmodule Mint.HTTP1Test do end end + describe "stream_headers option" do + setup %{port: port} do + assert {:ok, conn} = HTTP1.connect(:http, "localhost", port, stream_headers: true) + assert_receive {_server_ref, server_socket} + [conn: conn, server_socket: server_socket] + end + + test "emits all complete headers from a chunk at once when stream_headers is true", %{ + conn: conn + } do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + # Send two complete headers plus start of third header in one chunk + # This should emit the two complete headers together + assert {:ok, conn, [headers1]} = + HTTP1.stream(conn, {:tcp, conn.socket, "Foo: Bar\r\nBaz: Boz\r\nQux"}) + + assert {:headers, ^ref, [{"foo", "Bar"}, {"baz", "Boz"}]} = headers1 + + # Complete the third header and end headers section + assert {:ok, _conn, [headers2]} = + HTTP1.stream(conn, {:tcp, conn.socket, ": Quux\r\n\r\n"}) + + assert {:headers, ^ref, [{"qux", "Quux"}]} = headers2 + end + + test "emits multiple headers from one packet together", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + assert {:ok, _conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "Foo: Bar\r\nBaz: Boz\r\n\r\n"}) + + assert [headers] = responses + assert {:headers, ^ref, [{"foo", "Bar"}, {"baz", "Boz"}]} = headers + end + + test "handles partial headers with streaming", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + # Send first header with partial second header + assert {:ok, conn, [header1]} = HTTP1.stream(conn, {:tcp, conn.socket, "Foo: Bar\r\nB"}) + assert {:headers, ^ref, [{"foo", "Bar"}]} = header1 + + # Complete second header and end headers + assert {:ok, _conn, [header2]} = HTTP1.stream(conn, {:tcp, conn.socket, "az: Boz\r\n\r\n"}) + assert {:headers, ^ref, [{"baz", "Boz"}]} = header2 + end + + test "emits header immediately when packet ends exactly after LF", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + # Send a complete header ending exactly at packet boundary (no subsequent bytes) + assert {:ok, conn, responses} = + HTTP1.stream( + conn, + {:tcp, conn.socket, "X-Progress: 50\r\nX-Other: value\r\n"} + ) + + assert [{:headers, ^ref, [{"x-progress", "50"}, {"x-other", "value"}]}] = responses + + # Send another complete header ending exactly at packet boundary + assert {:ok, conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "X-Progress: 100\r\n"}) + + assert [{:headers, ^ref, [{"x-progress", "100"}]}] = responses + + # End the header section + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, "\r\n"}) + assert [] = responses + end + + test "streams trailer headers from same chunk together", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + assert {:ok, conn, [_headers]} = + HTTP1.stream(conn, {:tcp, conn.socket, "Transfer-Encoding: chunked\r\n\r\n"}) + + assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, "5\r\nhello\r\n"}) + assert [{:data, ^ref, "hello"}] = responses + + # Send last chunk and trailer headers in one chunk + assert {:ok, _conn, responses} = + HTTP1.stream( + conn, + {:tcp, conn.socket, "0\r\nX-Trailer-1: value1\r\nX-Trailer-2: value2\r\n\r\n"} + ) + + assert [trailers, done] = responses + assert {:headers, ^ref, [{"x-trailer-1", "value1"}, {"x-trailer-2", "value2"}]} = trailers + assert {:done, ^ref} = done + end + + test "filters unallowed trailer headers when streaming", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + assert {:ok, conn, [_headers]} = + HTTP1.stream(conn, {:tcp, conn.socket, "Transfer-Encoding: chunked\r\n\r\n"}) + + # Send last chunk with allowed and unallowed trailer headers + assert {:ok, _conn, responses} = + HTTP1.stream( + conn, + {:tcp, conn.socket, + "0\r\nContent-Length: 100\r\nX-Custom-Trailer: allowed\r\n\r\n"} + ) + + # Content-Length should be filtered out, only X-Custom-Trailer should appear + assert [trailer, done] = responses + assert {:headers, ^ref, [{"x-custom-trailer", "allowed"}]} = trailer + assert {:done, ^ref} = done + end + + test "emits trailer header immediately when packet ends exactly after LF", %{conn: conn} do + {:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil) + + assert {:ok, conn, [_status]} = + HTTP1.stream(conn, {:tcp, conn.socket, "HTTP/1.1 200 OK\r\n"}) + + assert {:ok, conn, [_headers]} = + HTTP1.stream(conn, {:tcp, conn.socket, "Transfer-Encoding: chunked\r\n\r\n"}) + + assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, "5\r\nhello\r\n"}) + assert [{:data, ^ref, "hello"}] = responses + + # Send last chunk (ending exactly at boundary) + assert {:ok, conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, "0\r\n"}) + assert [] = responses + + # Send a complete trailer header ending exactly at packet boundary (no subsequent bytes) + assert {:ok, conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "X-Trailer-1: value1\r\n"}) + + assert [{:headers, ^ref, [{"x-trailer-1", "value1"}]}] = responses + + # Send another complete trailer header ending exactly at packet boundary + assert {:ok, conn, responses} = + HTTP1.stream(conn, {:tcp, conn.socket, "X-Trailer-2: value2\r\n"}) + + assert [{:headers, ^ref, [{"x-trailer-2", "value2"}]}] = responses + + # End the trailer section + assert {:ok, _conn, responses} = HTTP1.stream(conn, {:tcp, conn.socket, "\r\n"}) + assert [{:done, ^ref}] = responses + end + end + @mint_user_agent "mint/#{Mix.Project.config()[:version]}" defp mint_user_agent, do: @mint_user_agent end