Merge pull request #4772 from barnabasdomozi/server_jobs_flag

Added flags to set server processes

Author: bruntib

docs/web/user_guide.md

@@ -186,6 +186,14 @@ optional arguments:
                         subsequent executions of the "same" server is achieved
                         in distinct environments, e.g., if the server
                         otherwise is running in a container.
+  --api-handler-processes API_HANDLER_PROCESSES
+                        Number of API request handler processes the server should start.
+                        This setting overrides the server config file values.
+  --task-worker-processes TASK_WORKER_PROCESSES
+                        Number of task worker processes the server should start.
+                        These handle report storage tasks (massStoreRun,
+                        massStoreRunAsynchronous).
+                        This setting overrides the server config file values.
   --host LISTEN_ADDRESS
                         The IP address or hostname of the server on which it
                         should listen for connections. For IPv6 listening,
@@ -256,6 +264,9 @@ By default, the running server can only be accessed from the same machine
 (`localhost`) where it is running. This can be overridden by specifying
 `--host ""`, instructing the server to listen on all available interfaces.
 
+The server spawns additional Python processes to handle incoming API requests
+and to process report storing tasks. The process counts can be changed using
+the `--api-handler-processes` and `--task-worker-processes` flags, respectively.
 
 #### Run CodeChecker server in Docker
 To run CodeChecker server in Docker see the [Docker](docker.md) documentation.
@@ -1796,4 +1807,4 @@ In JSON output we have two main sections:
 
 ## Log Levels
 
-To change the log levels check out the [logging](../logging.md) documentation.
+To change the log levels check out the [logging](../logging.md) documentation.

web/server/codechecker_server/api/report_server.py

@@ -48,6 +48,8 @@
     SourceComponentData, SourceFileData, SortMode, SortType, \
     SubmittedRunOptions
 
+from codechecker_api_shared.ttypes import ErrorCode, RequestFailed
+
 from codechecker_common import util
 from codechecker_common.logger import get_logger
 
@@ -3999,6 +4001,10 @@ def __massStoreRun_common(self, is_async: bool, zipfile_blob: str,
         if not store_opts.runName:
             raise ValueError("A run name is needed to know where to store!")
 
+        if self._manager.background_worker_processes == 0:
+            raise RequestFailed(ErrorCode.GENERAL,
+                                "No task worker process is available!")
+
         from .mass_store_run import MassStoreRunInputHandler, MassStoreRunTask
         ih = MassStoreRunInputHandler(self._manager,
                                       self._config_database,

web/server/codechecker_server/cli/server.py

@@ -113,6 +113,25 @@ def add_arguments_to_parser(parser):
 In most scenarios, there is no need to fine-tune this, except if subsequent
 executions of the "same" server is achieved in distinct environments, e.g.,
 if the server otherwise is running in a container.
+""")
+
+    parser.add_argument("--api-handler-processes",
+                        type=int,
+                        dest="api_handler_processes",
+                        required=False,
+                        help="""
+Number of API request handler processes the server should start.
+This setting overrides the server config file values.
+""")
+
+    parser.add_argument("--task-worker-processes",
+                        type=int,
+                        dest="task_worker_processes",
+                        required=False,
+                        help="""
+Number of task worker processes the server should start.
+These handle report storage tasks (massStoreRun, massStoreRunAsynchronous).
+This setting overrides the server config file values.
 """)
 
     parser.add_argument('--host',
@@ -1043,7 +1062,9 @@ def server_init_start(args):
                                    args.skip_db_cleanup,
                                    context,
                                    environ,
-                                   machine_id)
+                                   machine_id,
+                                   args.api_handler_processes,
+                                   args.task_worker_processes)
     except socket.error as err:
         if err.errno == errno.EADDRINUSE:
             LOG.error("Server can't be started, maybe port number (%s) is "

web/server/codechecker_server/server.py

@@ -992,7 +992,9 @@ def start_server(config_directory: str, workspace_directory: str,
                  package_data, port: int, config_sql_server,
                  listen_address: str, force_auth: bool,
                  skip_db_cleanup: bool, context, check_env,
-                 machine_id: str) -> int:
+                 machine_id: str,
+                 api_handler_processes: Optional[int],
+                 task_worker_processes: Optional[int]) -> int:
     """
     Starts the HTTP server to handle Web client and Thrift requests, execute
     background jobs.
@@ -1036,7 +1038,9 @@ def start_server(config_directory: str, workspace_directory: str,
         manager = session_manager.SessionManager(
             server_cfg_file,
             server_secrets_file,
-            force_auth)
+            force_auth,
+            api_handler_processes,
+            task_worker_processes)
 
     except IOError as ioerr:
         LOG.debug(ioerr)
@@ -1067,8 +1071,7 @@ def start_server(config_directory: str, workspace_directory: str,
 
     bg_processes: Dict[int, Process] = {}
     requested_bg_threads = cast(int,
-                                manager.background_worker_processes) \
-        or requested_api_threads
+                                manager.background_worker_processes)
     # Note that Queue under the hood uses OS-level primitives such as a socket
     # or a pipe, where the read-write buffers have a **LIMITED** capacity, and
     # are usually **NOT** backed by the full amount of available system memory.
@@ -1220,7 +1223,7 @@ def spawn_bg_process():
         signal_log(LOG, "DEBUG", f"Task child process {p.pid} started!")
         return p
 
-    LOG.info("Using %d Task handler processes ...", requested_bg_threads)
+    LOG.info("Using %d Task worker processes ...", requested_bg_threads)
     for _ in range(requested_bg_threads):
         spawn_bg_process()
 

web/server/codechecker_server/session_manager.py

@@ -171,7 +171,9 @@ class SessionManager:
     CodeChecker server.
     """
 
-    def __init__(self, configuration_file, secrets_file, force_auth=False):
+    def __init__(self, configuration_file, secrets_file, force_auth=False,
+                 api_handler_processes: Optional[int] = None,
+                 task_worker_processes: Optional[int] = None):
         """
         Initialise a new Session Manager on the server.
 
@@ -194,6 +196,13 @@ def __init__(self, configuration_file, secrets_file, force_auth=False):
         # instantiate SessionManager with the found configuration.
         self.__worker_processes, self.__background_worker_processes = \
             get_worker_processes(self.scfg_dict)
+
+        if api_handler_processes is not None:
+            self.__worker_processes = api_handler_processes
+
+        if task_worker_processes is not None:
+            self.__background_worker_processes = task_worker_processes
+
         self.__max_run_count = self.scfg_dict.get('max_run_count', None)
         self.__store_config = self.scfg_dict.get('store', {})
         self.__keepalive_config = self.scfg_dict.get('keepalive', {})