Repository navigation
http_server: handle pipelined HTTP/1.1 requests already in the buffer #12794
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
3fae365
4755c91
3be7a89
cb417e6
86825ad
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -544,6 +544,7 @@ static int flb_http_server_client_activity_event_handler(void *data) | |
|
|
||
| close_connection = FLB_FALSE; | ||
|
|
||
| dispatch_requests: | ||
| cfl_list_foreach_safe(iterator, | ||
| backup_iterator, | ||
| &session->request_queue) { | ||
|
|
@@ -602,6 +603,31 @@ static int flb_http_server_client_activity_event_handler(void *data) | |
| } | ||
| } | ||
|
|
||
| /* | ||
| * An HTTP/1.x session parses one request at a time, and the bytes of a | ||
| * pipelined request that arrived in the same read stay in incoming_data. | ||
| * No read event will come for them, so parse them now instead of | ||
| * leaving the client waiting until it sends more data. | ||
| */ | ||
| if (!close_connection && | ||
| session->version != HTTP_PROTOCOL_VERSION_AUTODETECT && | ||
| session->version <= HTTP_PROTOCOL_VERSION_11 && | ||
| cfl_sds_len(session->incoming_data) > 0) { | ||
| result = flb_http1_server_session_ingest(&session->http1, NULL, 0); | ||
|
|
||
| if (result < 0) { | ||
| /* Send the responses of the requests already handled */ | ||
| flb_http_server_session_write(session); | ||
| flb_http_server_session_destroy(session); | ||
|
|
||
| return -1; | ||
| } | ||
|
|
||
| if (!cfl_list_is_empty(&session->request_queue)) { | ||
| goto dispatch_requests; | ||
| } | ||
| } | ||
|
|
||
| result = flb_http_server_session_write(session); | ||
|
|
||
| if (result != 0) { | ||
|
Comment on lines
603
to
633
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Set the buffer length after the remainder moves to the front. When one read contains two complete HTTP/1.x requests, the first request is evicted and the second request moves to the front of Set the header length to the remainder length after the Suggested fix
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| """Pipelined HTTP/1.1 requests sent in a single write are all answered.""" | ||
|
|
||
| import socket | ||
|
|
||
| import pytest | ||
|
|
||
| from utils.test_service import FluentBitTestService | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def listener(tmp_path): | ||
| config = tmp_path / "http1_pipelining.yaml" | ||
| config.write_text("""service: | ||
| flush: 1 | ||
| grace: 1 | ||
| http_server: on | ||
| http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} | ||
| pipeline: | ||
| inputs: | ||
| - name: http | ||
| listen: 127.0.0.1 | ||
| port: ${FLUENT_BIT_TEST_LISTENER_PORT} | ||
| enable_health_endpoint: on | ||
| outputs: | ||
| - name: null | ||
| match: '*' | ||
| """) | ||
| service = FluentBitTestService(str(config)) | ||
| service.start() | ||
| try: | ||
| yield service.flb_listener_port | ||
| finally: | ||
| service.stop() | ||
|
|
||
|
|
||
| def read_response(stream): | ||
| """Read one response from a buffered socket file, return its status.""" | ||
| status_line = stream.readline() | ||
| assert status_line, "connection closed before a response arrived" | ||
| status = int(status_line.split()[1]) | ||
|
|
||
| length = 0 | ||
| chunked = False | ||
| while True: | ||
| line = stream.readline() | ||
| if line in (b"\r\n", b"\n", b""): | ||
| break | ||
| name, _, value = line.decode("latin-1").partition(":") | ||
| if name.strip().lower() == "content-length": | ||
| length = int(value.strip()) | ||
| elif name.strip().lower() == "transfer-encoding" and "chunked" in value.lower(): | ||
| chunked = True | ||
|
|
||
| if chunked: | ||
| while True: | ||
| size = int(stream.readline().split(b";")[0], 16) | ||
| stream.read(size + 2) | ||
| if size == 0: | ||
| break | ||
| else: | ||
| stream.read(length) | ||
|
|
||
| return status | ||
|
|
||
|
|
||
| def test_http1_pipelined_requests_in_one_write(listener): | ||
| # A chunked body has an exact end. With Content-Length the body currently | ||
| # runs to the end of the buffer (#12652), which is a separate bug. | ||
| body = b'{"a":1}' | ||
| first = (b"POST /one HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" | ||
| b"Transfer-Encoding: chunked\r\n\r\n" | ||
| + f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n") | ||
| second = b"GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n" | ||
|
|
||
| with socket.create_connection(("127.0.0.1", listener), timeout=5) as sock: | ||
| # Both requests in one write, and nothing sent after them: the second | ||
| # one must be answered without waiting for more data from the client. | ||
| sock.sendall(first + second) | ||
| stream = sock.makefile("rb") | ||
| assert read_response(stream) == 201 | ||
| assert read_response(stream) == 200 | ||
|
|
||
|
|
||
| def test_http1_pipelined_header_only_requests(listener): | ||
| # Two requests without a body. The first one ends at its blank line, so | ||
| # the second one has to be kept in the buffer when the first is evicted. | ||
| request = b"GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n" | ||
|
|
||
| with socket.create_connection(("127.0.0.1", listener), timeout=5) as sock: | ||
| sock.sendall(request + request) | ||
| stream = sock.makefile("rb") | ||
| assert read_response(stream) == 200 | ||
| assert read_response(stream) == 200 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a header-only request is first in the pipeline (for example, two GETs in one write),
http1_evict_request()gets(size_t)-1frommk_http_parser_request_size()for a non-chunked request without a positive Content-Length and clearsincoming_dataatflb_http_server_http1.c:79-83. This guard is therefore false after the first dispatch and the second request has already been discarded, so the change still returns only one response for the most common HTTP pipeline; the new test avoids this path by making the first request a chunked POST. Preserve the bytes after the first header and cover GET/GET before relying on this length check.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Right, the evict step dropped everything after a request with no body. It now ends that request at the blank line after its headers and keeps the rest, plus a GET/GET test. Fixed in 3be7a89.