BUG: RuntimeError: deque mutated during iteration #3633
Description
The starting point for issues should usually be a discussion...
https://github.com/encode/httpx/discussions/
Possible bugs may be raised as a "Potential Issue" discussion, feature requests may be raised as an "Ideas" discussion. We can then determine if the discussion needs to be escalated into an "Issue" or not.
This will help us ensure that the "Issues" list properly reflects ongoing or needed work on the project.
- Initially raised as discussion RuntimeError: deque mutated during iteration (hpack) #3279, but no response/action taken
Exception raised by this bug; it's already been discussed at python-hyper/hpack#275 and appears from time to time in a function that is decorated by backoff 2.2.1 (https://pypi.org/project/backoff/):
Backing off get_with_retry(...) for 0.1s (httpx.RemoteProtocolError: <ConnectionTerminated error_code:9, last_stream_id:15, additional_data:None>)
Backing off get_with_retry(...) for 0.4s (httpx.RemoteProtocolError: <ConnectionTerminated error_code:9, last_stream_id:15, additional_data:None>)
Traceback (most recent call last):
File "/host/usr/local/bin/watch.py", line 567, in <module>
main()
File "/host/usr/local/bin/watch.py", line 396, in main
for _ in executor.map(download, secs, repeat(c), repeat(ret_periods), repeat(display_price)):
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 619, in result_iterator
yield _result_or_cancel(fs.pop())
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 317, in _result_or_cancel
return fut.result(timeout)
^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/host/usr/local/bin/watch.py", line 145, in download
resp = get_with_retry(url, c, timeout=5)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/backoff/_sync.py", line 105, in retry
ret = target(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/backoff/_sync.py", line 48, in retry
ret = target(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/host/usr/local/bin/mb_httpx.py", line 104, in get_with_retry
return c.get(
^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 1054, in get
return self.request(
^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 827, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 914, in send
response = self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_client.py", line 1015, in _send_single_request
response = transport.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpx/_transports/default.py", line 233, in handle_request
resp = self._pool.handle_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/connection_pool.py", line 216, in handle_request
raise exc from None
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/connection_pool.py", line 196, in handle_request
response = connection.handle_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/connection.py", line 101, in handle_request
return self._connection.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/http2.py", line 185, in handle_request
raise exc
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/http2.py", line 142, in handle_request
self._send_request_headers(request=request, stream_id=stream_id)
File "/usr/local/lib/python3.12/dist-packages/httpcore/_sync/http2.py", line 247, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
File "/usr/local/lib/python3.12/dist-packages/h2/connection.py", line 770, in send_headers
frames = stream.send_headers(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/h2/stream.py", line 867, in send_headers
frames = self._build_headers_frames(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/h2/stream.py", line 1254, in _build_headers_frames
encoded_headers = encoder.encode(headers)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/hpack/hpack.py", line 255, in encode
header_block.append(self.add(header, sensitive, huffman))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/hpack/hpack.py", line 280, in add
match = self.header_table.search(name, value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/hpack/table.py", line 184, in search
for (i, (n, v)) in enumerate(self.dynamic_entries):
RuntimeError: deque mutated during iteration
Again, also discussed at python-hyper/hpack#281
RuntimeError: deque mutated during iteration
Traceback (most recent call last):
File "/workspace/main.py", line 197, in get_status
return get_with_retry_5s(url, c)
~~~~~~~~~~~~^^^^^^^^^
File "/workspace/main.py", line 616, in uaflifo_main
inb_status, _, _ = get_flifo_data(new_args, c=c)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
File "/workspace/main.py", line 223, in get_flifo_data
fs_response = get_flight_status(args, c)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/backoff/_sync.py", line 105, in retry
ret = target(*args, **kwargs)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/backoff/_sync.py", line 48, in retry
ret = target(*args, **kwargs)
File "/workspace/mb_httpx.py", line 213, in get_with_retry_5s
return c.get(
~~~~~^
url,
^^^^
...<6 lines>...
extensions=extensions,
^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 1053, in get
return self.request(
~~~~~~~~~~~~^
"GET",
^^^^^^
...<7 lines>...
extensions=extensions,
^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 825, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 914, in send
response = self._send_handling_auth(
request,
...<2 lines>...
history=[],
)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
request,
follow_redirects=follow_redirects,
history=history,
)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpx/_transports/default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/connection_pool.py", line 256, in handle_request
raise exc from None
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/connection_pool.py", line 236, in handle_request
response = connection.handle_request(
pool_request.request
)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/connection.py", line 103, in handle_request
return self._connection.handle_request(request)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/http2.py", line 187, in handle_request
raise exc
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/http2.py", line 144, in handle_request
self._send_request_headers(request=request, stream_id=stream_id)
~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/httpcore/_sync/http2.py", line 249, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/h2/connection.py", line 806, in send_headers
frames.extend(stream.send_headers(
~~~~~~~~~~~~~~~~~~~^
headers, self.encoder, end_stream,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
))
^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/h2/stream.py", line 894, in send_headers
frames = self._build_headers_frames(
bytes_headers, encoder, hf, hdr_validation_flags,
)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/h2/stream.py", line 1298, in _build_headers_frames
encoded_headers = encoder.encode(headers)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/hpack/hpack.py", line 276, in encode
header_block.append(self.add(new_header, sensitive, huffman))
~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/hpack/hpack.py", line 301, in add
match = self.header_table.search(name, value)
File "/layers/google.python.pip/pip/lib/python3.13/site-packages/hpack/table.py", line 186, in search
for (i, (n, v)) in enumerate(self.dynamic_entries):
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
The code (in mb_httpx.py) that (rarely) causes httpx to crash is the following:
@backoff.on_exception(
wait_gen=backoff.expo,
exception=httpx.HTTPError,
max_time=5,
on_backoff=log_backoff,
)
@backoff.on_predicate(
wait_gen=backoff.expo,
predicate=lambda x: x.status_code in {408, 429, 500, 502, 503, 504},
max_time=5,
on_backoff=log_backoff,
)
def get_with_retry_5s(
url: httpx._types.URLTypes,
c: httpx.Client = default_client,
*,
params: httpx._types.QueryParamTypes | None = None,
headers: httpx._types.HeaderTypes | None = None,
cookies: httpx._types.CookieTypes | None = None,
auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT,
follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT,
timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT,
extensions: httpx._types.RequestExtensions | None = None,
) -> httpx.Response:
"""
Get url, retrying for up to 5 seconds with truncated exponential backoff if an HTTPError is encountered or
any of the following response status codes is returned:
* 408 Request Timeout
* 429 Too Many Requests
* 500 Internal Server Error
* 502 Bad Gateway
* 503 Service Unavailable
* 504 Gateway Timeout
"""
timeout = timeout or 5
return c.get(
url,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
From the logs, coincidental to the Exception is the recording of a httpx.ReadError: [Errno 11] Resource temporarily unavailable) by log_backoff (the logger from encountering a backoff condition), but I cannot determine whether this takes place before the Exception or is simply a byproduct of it.