dagda crashing when sent check job?

[zestysoft]

Short description

Check image

Reproduction steps

1. docker-compose up
2. python3 dagda.py check --docker_image prizmi/learner:0.93

Expected results

Job completes without crashing

Actual results

(dagda) user@ubuntu:~/github/dagda/dagda$ python3 dagda.py check --docker_image prizmi/learner:0.93
{
    "id": "624f384aece913ec370b4739",
    "msg": "Accepted the analysis of <prizmi/learner:0.93>"
}
(dagda) user@ubuntu:~/github/dagda/dagda$ python3 dagda.py history prizmi/learner:0.93 --id 624f384aece913ec370b4739
[
    {
        "id": "624f384aece913ec370b4739",
        "image_name": "prizmi/learner:0.93",
        "status": "Analyzing",
        "timestamp": "2022-04-07 19:15:22.071120"
    }
]
(dagda) user@ubuntu:~/github/dagda/dagda$ python3 dagda.py history prizmi/learner:0.93 --id 624f384aece913ec370b4739
{
    "err": 500,
    "msg": "Internal Server Error"
}

On which platforms did you notice this:

Please complete the following information:

• OS: Ubuntu
• OS Version: Linux ubuntu 5.4.0-107-generic #121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
• Python version: Python 3.8.10
• Docker version 20.10.14

Backtrace

dagda     | <2022-04-07 19:16:09,466> <ERROR> <DagdaServer> <analyzer> <evaluate_image:102> <Unexpected exception of type APIError occurred: (HTTPError('500 Server Error: Internal Server Error for url: http+docker://localhost/v1.41/images/prizmi/learner:0.93/get'),)>
dagda     | <2022-04-07 19:16:34,670> <ERROR> <DagdaServer> <dagda> <main:46> <vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>>
dagda     | Traceback (most recent call last):
dagda     |   File "dagda.py", line 35, in main
dagda     |     r = execute_dagda_cmd(cmd=cmd, args=parsed_args)
dagda     |   File "/opt/app/cli/dagda_cli.py", line 43, in execute_dagda_cmd
dagda     |     ds.run()
dagda     |   File "/opt/app/api/dagda_server.py", line 93, in run
dagda     |     docker_daemon_events_monitor.run()
dagda     |   File "/opt/app/analysis/runtime/docker_events_monitor.py", line 53, in run
dagda     |     self.mongodb_driver.bulk_insert_docker_daemon_events([e])
dagda     |   File "/opt/app/driver/mongodb_driver.py", line 134, in bulk_insert_docker_daemon_events
dagda     |     self.db.docker_events.create_index([('from', pymongo.DESCENDING)])
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/collection.py", line 2100, in create_index
dagda     |     return self.__create_indexes([index], session, **cmd_options)[0]
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/collection.py", line 1960, in __create_indexes
dagda     |     with self._socket_for_writes(session) as sock_info:
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/collection.py", line 198, in _socket_for_writes
dagda     |     return self.__database.client._socket_for_writes(session)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1356, in _socket_for_writes
dagda     |     server = self._select_server(writable_server_selector, session)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1346, in _select_server
dagda     |     server = topology.select_server(server_selector)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 244, in select_server
dagda     |     return random.choice(self.select_servers(selector,
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 202, in select_servers
dagda     |     server_descriptions = self._select_servers_loop(
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 218, in _select_servers_loop
dagda     |     raise ServerSelectionTimeoutError(
dagda     | pymongo.errors.ServerSelectionTimeoutError: vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>
dagda     | <2022-04-07 19:16:39,489> <ERROR> <DagdaServer> <dagda> <main:46> <vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>>
dagda     | Traceback (most recent call last):
dagda     |   File "dagda.py", line 35, in main
dagda     |     r = execute_dagda_cmd(cmd=cmd, args=parsed_args)
dagda     |   File "/opt/app/cli/dagda_cli.py", line 43, in execute_dagda_cmd
dagda     |     ds.run()
dagda     |   File "/opt/app/api/dagda_server.py", line 81, in run
dagda     |     self._check_docker_by_image_name(item)
dagda     |   File "/opt/app/api/dagda_server.py", line 166, in _check_docker_by_image_name
dagda     |     evaluated_docker_image = analyzer.evaluate_image(item['image_name'], None)
dagda     |   File "/opt/app/analysis/analyzer.py", line 120, in evaluate_image
dagda     |     data['static_analysis'] = self.generate_static_analysis(image_name, os_packages, dependencies, malware_binaries)
dagda     |   File "/opt/app/analysis/analyzer.py", line 134, in generate_static_analysis
dagda     |     data['os_packages'] = self.generate_os_report(image_name, os_packages)
dagda     |   File "/opt/app/analysis/analyzer.py", line 180, in generate_os_report
dagda     |     p['vulnerabilities'] = self.get_vulnerabilities(package['product'], package['version'])
dagda     |   File "/opt/app/analysis/analyzer.py", line 202, in get_vulnerabilities
dagda     |     return self.mongoDbDriver.get_vulnerabilities(product, version)
dagda     |   File "/opt/app/driver/mongodb_driver.py", line 305, in get_vulnerabilities
dagda     |     for cve in cve_cursor:
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1238, in next
dagda     |     if len(self.__data) or self._refresh():
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1130, in _refresh
dagda     |     self.__session = self.__collection.database.client._ensure_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1935, in _ensure_session
dagda     |     return self.__start_session(True, causal_consistency=False)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1883, in __start_session
dagda     |     server_session = self._get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1921, in _get_server_session
dagda     |     return self._topology.get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 520, in get_server_session
dagda     |     session_timeout = self._check_session_support()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 499, in _check_session_support
dagda     |     self._select_servers_loop(
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 218, in _select_servers_loop
dagda     |     raise ServerSelectionTimeoutError(
dagda     | pymongo.errors.ServerSelectionTimeoutError: vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>
dagda     | <2022-04-07 19:47:27,404> <ERROR> <DagdaServer> <app> <log_exception:1454> <Exception on /v1/history/prizmi/learner:0.93 [GET]>
dagda     | Traceback (most recent call last):
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 2070, in wsgi_app
dagda     |     response = self.full_dispatch_request()
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1515, in full_dispatch_request
dagda     |     rv = self.handle_user_exception(e)
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask_cors/extension.py", line 165, in wrapped_function
dagda     |     return cors_after_request(app.make_response(f(*args, **kwargs)))
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1513, in full_dispatch_request
dagda     |     rv = self.dispatch_request()
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1499, in dispatch_request
dagda     |     return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
dagda     |   File "/opt/app/api/service/history.py", line 43, in get_history_by_image_name
dagda     |     history = InternalServer.get_mongodb_driver().get_docker_image_history(image_name, id)
dagda     |   File "/opt/app/driver/mongodb_driver.py", line 523, in get_docker_image_history
dagda     |     for scan in cursor:
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1238, in next
dagda     |     if len(self.__data) or self._refresh():
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1130, in _refresh
dagda     |     self.__session = self.__collection.database.client._ensure_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1935, in _ensure_session
dagda     |     return self.__start_session(True, causal_consistency=False)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1883, in __start_session
dagda     |     server_session = self._get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1921, in _get_server_session
dagda     |     return self._topology.get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 520, in get_server_session
dagda     |     session_timeout = self._check_session_support()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 499, in _check_session_support
dagda     |     self._select_servers_loop(
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 218, in _select_servers_loop
dagda     |     raise ServerSelectionTimeoutError(
dagda     | pymongo.errors.ServerSelectionTimeoutError: vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>
dagda     | <2022-04-07 19:48:24,973> <ERROR> <DagdaServer> <app> <log_exception:1454> <Exception on /v1/history/prizmi/learner:0.93 [GET]>
dagda     | Traceback (most recent call last):
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 2070, in wsgi_app
dagda     |     response = self.full_dispatch_request()
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1515, in full_dispatch_request
dagda     |     rv = self.handle_user_exception(e)
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask_cors/extension.py", line 165, in wrapped_function
dagda     |     return cors_after_request(app.make_response(f(*args, **kwargs)))
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1513, in full_dispatch_request
dagda     |     rv = self.dispatch_request()
dagda     |   File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1499, in dispatch_request
dagda     |     return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
dagda     |   File "/opt/app/api/service/history.py", line 43, in get_history_by_image_name
dagda     |     history = InternalServer.get_mongodb_driver().get_docker_image_history(image_name, id)
dagda     |   File "/opt/app/driver/mongodb_driver.py", line 523, in get_docker_image_history
dagda     |     for scan in cursor:
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1238, in next
dagda     |     if len(self.__data) or self._refresh():
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/cursor.py", line 1130, in _refresh
dagda     |     self.__session = self.__collection.database.client._ensure_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1935, in _ensure_session
dagda     |     return self.__start_session(True, causal_consistency=False)
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1883, in __start_session
dagda     |     server_session = self._get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1921, in _get_server_session
dagda     |     return self._topology.get_server_session()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 520, in get_server_session
dagda     |     session_timeout = self._check_session_support()
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 499, in _check_session_support
dagda     |     self._select_servers_loop(
dagda     |   File "/usr/local/lib/python3.8/site-packages/pymongo/topology.py", line 218, in _select_servers_loop
dagda     |     raise ServerSelectionTimeoutError(
dagda     | pymongo.errors.ServerSelectionTimeoutError: vulndb:27017: [Errno -3] Try again, Timeout: 30s, Topology Description: <TopologyDescription id: 624f376dece913ec370b4738, topology_type: Single, servers: [<ServerDescription ('vulndb', 27017) server_type: Unknown, rtt: None, error=AutoReconnect('vulndb:27017: [Errno -3] Try again')>]>