Skip to content

Handle upstream server errors gracefully (and not die) #1

@torfinnnome

Description

@torfinnnome
$ bin/eccd.py ecc.yam
[...]
nodes_total: 1, nodes_idle: 1, jobs_pending: 0
2021-06-25 03:09:21 eccd.INFO     Nothing to change.
2021-06-25 03:09:21 eccd.INFO     Napping for 30 seccnd(s).
Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 170, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 73, in create_connection
    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
  File "/usr/lib64/python3.6/socket.py", line 745, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 706, in urlopen
    chunked=chunked,
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 382, in _make_request
    self._validate_conn(conn)
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 1010, in _validate_conn
    conn.connect()
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 353, in connect
    conn = self._new_conn()
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 182, in _new_conn
    self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7febb118c208>: Failed to establish a new connection: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 756, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 574, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c208>: Failed to establish a new connection: [Errno -2] Name or service not known',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 1022, in _send_request
    resp = self.session.request(method, url, **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/adapters.py", line 516, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c208>: Failed to establish a new connection: [Errno -2] Name or service not known',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 1038, in _send_request
    raise exceptions.ConnectFailure(msg)
keystoneauth1.exceptions.connection.ConnectFailure: Unable to establish connection to https://compute.api.bgo.nrec.no:8774/v2.1/servers/detail: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c208>: Failed to establish a new connection: [Errno -2] Name or service not known',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 170, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 73, in create_connection
    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
  File "/usr/lib64/python3.6/socket.py", line 745, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 706, in urlopen
    chunked=chunked,
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 382, in _make_request
    self._validate_conn(conn)
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 1010, in _validate_conn
    conn.connect()
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 353, in connect
    conn = self._new_conn()
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connection.py", line 182, in _new_conn
    self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7febb118c390>: Failed to establish a new connection: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 756, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "/srv/ecc/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 574, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c390>: Failed to establish a new connection: [Errno -2] Name or service not known',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 1022, in _send_request
    resp = self.session.request(method, url, **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/requests/adapters.py", line 516, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c390>: Failed to establish a new connection: [Errno -2] Name or service not known',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "bin/eccd.py", line 109, in <module>
    main()
  File "bin/eccd.py", line 105, in main
    run_daemon()
  File "bin/eccd.py", line 56, in run_daemon
    ecc.update_nodes_status()
  File "/srv/ecc/venv/lib/python3.6/site-packages/ecc/ecc.py", line 65, in update_nodes_status
    vnodes = servers(config.ecc.name_regex)
  File "/srv/ecc/venv/lib/python3.6/site-packages/ecc/ecc.py", line 49, in servers
    servers = openstack.servers()
  File "/srv/ecc/venv/lib/python3.6/site-packages/ecc/openstack_class.py", line 151, in servers
    for server in self._connection.compute.servers():
  File "/srv/ecc/venv/lib/python3.6/site-packages/openstack/resource.py", line 1753, in list
    microversion=microversion)
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/adapter.py", line 395, in get
    return self.request(url, 'GET', **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/openstack/proxy.py", line 100, in request
    **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/adapter.py", line 257, in request
    return self.session.request(url, method, **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 931, in request
    resp = send(**kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 1061, in _send_request
    **kwargs)
  File "/srv/ecc/venv/lib/python3.6/site-packages/keystoneauth1/session.py", line 1038, in _send_request
    raise exceptions.ConnectFailure(msg)
keystoneauth1.exceptions.connection.ConnectFailure: Unable to establish connection to https://compute.api.bgo.nrec.no:8774/v2.1/servers/detail: HTTPSConnectionPool(host='compute.api.bgo.nrec.no', port=8774): Max retries exceeded with url: /v2.1/servers/detail (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7febb118c390>: Failed to establish a new connection: [Errno -2] Name or service not known',))
$

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions