An error during fee rate retrieval causes socket to crash

[bboerst]

Hit a scenario where table stops updating. Looked at the webapp logs and see the errors below.

I think in this case getting the fee rate from mempool timed out and then immediately after a ConnectionResetError(104, 'Connection reset by peer'); sock=<eventlet.greenio.base.GreenSocket object at 0x7f9b197110>

ERROR:main:Error fetching transaction fee rate for c6eb7ab8dfb18eddb06fb3b91fa06844354f1d292d7f70636b0b63b463b58aa2
Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 467, in _make_request
    self._validate_conn(conn)
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 1099, in _validate_conn
    conn.connect()
  File "/usr/local/lib/python3.12/site-packages/urllib3/connection.py", line 653, in connect
    sock_and_verified = _ssl_wrap_socket_and_match_hostname(
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/connection.py", line 806, in _ssl_wrap_socket_and_match_hostname
    ssl_sock = ssl_wrap_socket(
               ^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py", line 465, in ssl_wrap_socket
    ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/util/ssl_.py", line 509, in _ssl_wrap_socket_impl
    return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/green/ssl.py", line 436, in wrap_socket
    return GreenSSLSocket(sock, *a, _context=self, **kw)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/green/ssl.py", line 144, in __init__
    self.do_handshake()
  File "/usr/local/lib/python3.12/site-packages/eventlet/green/ssl.py", line 324, in do_handshake
    return self._call_trampolining(
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/green/ssl.py", line 166, in _call_trampolining
    return func(*a, **kw)
           ^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/ssl.py", line 1320, in do_handshake
    self._sslobj.do_handshake()
TimeoutError: [Errno 110] Connection timed out

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/requests/adapters.py", line 667, in send
    resp = conn.urlopen(
           ^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 847, in urlopen
    retries = retries.increment(
              ^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/util/retry.py", line 470, in increment
    raise reraise(type(error), error, _stacktrace)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/util/util.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 793, in urlopen
    response = self._make_request(
               ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 491, in _make_request
    raise new_e
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 469, in _make_request
    self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
  File "/usr/local/lib/python3.12/site-packages/urllib3/connectionpool.py", line 370, in _raise_timeout
    raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='mempool.space', port=443): Read timed out. (read timeout=None)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/app/main.py", line 121, in get_transaction_fee_rate
    response = requests.get(f'https://mempool.space/api/tx/{first_transaction}')
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/requests/api.py", line 73, in get
    return request("get", url, params=params, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/requests/api.py", line 59, in request
    return session.request(method=method, url=url, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 589, in request
    resp = self.send(prep, **send_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/requests/sessions.py", line 703, in send
    r = adapter.send(request, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/requests/adapters.py", line 713, in send
    raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='mempool.space', port=443): Read timed out. (read timeout=None)
ERROR:pika.adapters.utils.io_services_utils:_AsyncBaseTransport._consume() failed, aborting connection: error=ConnectionResetError(104, 'Connection reset by peer'); sock=<eventlet.greenio.base.GreenSocket object at 0x7f9b197110>; Caller's stack:
Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 1047, in _on_socket_readable
    self._consume()
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 791, in _consume
    data = self._sigint_safe_recv(self._sock, self._MAX_RECV_BYTES)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 79, in retry_sigint_wrap
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 846, in _sigint_safe_recv
    return sock.recv(max_bytes)
           ^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/greenio/base.py", line 352, in recv
    return self._recv_loop(self.fd.recv, b'', bufsize, flags)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/greenio/base.py", line 322, in _recv_loop
    return recv_meth(*args)
           ^^^^^^^^^^^^^^^^
ConnectionResetError: [Errno 104] Connection reset by peer
Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 1047, in _on_socket_readable
    self._consume()
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 791, in _consume
    data = self._sigint_safe_recv(self._sock, self._MAX_RECV_BYTES)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 79, in retry_sigint_wrap
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/utils/io_services_utils.py", line 846, in _sigint_safe_recv
    return sock.recv(max_bytes)
           ^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/greenio/base.py", line 352, in recv
    return self._recv_loop(self.fd.recv, b'', bufsize, flags)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/eventlet/greenio/base.py", line 322, in _recv_loop
    return recv_meth(*args)
           ^^^^^^^^^^^^^^^^
ConnectionResetError: [Errno 104] Connection reset by peer
INFO:pika.adapters.utils.io_services_utils:_AsyncTransportBase._initate_abort(): Initiating abrupt asynchronous transport shutdown: state=1; error=ConnectionResetError(104, 'Connection reset by peer'); <eventlet.greenio.base.GreenSocket object at 0x7f9b197110>
INFO:pika.adapters.utils.io_services_utils:Deactivating transport: state=1; <eventlet.greenio.base.GreenSocket object at 0x7f9b197110>
ERROR:pika.adapters.base_connection:connection_lost: StreamLostError: ("Stream connection lost: ConnectionResetError(104, 'Connection reset by peer')",)
INFO:pika.connection:AMQP stack terminated, failed to connect, or aborted: opened=True, error-arg=StreamLostError: ("Stream connection lost: ConnectionResetError(104, 'Connection reset by peer')",); pending-error=None
INFO:pika.connection:Stack terminated due to StreamLostError: ("Stream connection lost: ConnectionResetError(104, 'Connection reset by peer')",)
INFO:pika.adapters.utils.io_services_utils:Closing transport socket and unlinking: state=2; <eventlet.greenio.base.GreenSocket object at 0x7f9b197110>
ERROR:pika.adapters.blocking_connection:Unexpected connection close detected: StreamLostError: ("Stream connection lost: ConnectionResetError(104, 'Connection reset by peer')",)
Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/eventlet/hubs/poll.py", line 111, in wait
    listener.cb(fileno)
  File "/usr/local/lib/python3.12/site-packages/eventlet/green/select.py", line 53, in on_read
    current.switch(([original], [], []))
  File "/usr/local/lib/python3.12/site-packages/eventlet/greenthread.py", line 265, in main
    result = function(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/main.py", line 188, in consume_messages
    channel.start_consuming()
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/blocking_connection.py", line 1883, in start_consuming
    self._process_data_events(time_limit=None)
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/blocking_connection.py", line 2044, in _process_data_events
    self.connection.process_data_events(time_limit=time_limit)
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/blocking_connection.py", line 842, in process_data_events
    self._flush_output(common_terminator)
  File "/usr/local/lib/python3.12/site-packages/pika/adapters/blocking_connection.py", line 523, in _flush_output
    raise self._closed_result.value.error
pika.exceptions.StreamLostError: Stream connection lost: ConnectionResetError(104, 'Connection reset by peer')
Removing descriptor: 17