Parallelize library builder discovery phase

bin/lib/ce_install.py

@@ -758,6 +758,12 @@ def install(context: CliContext, filter_: list[str], force: bool):
 )
 @click.option("--popular-compilers-only", is_flag=True, help="Only build with popular (enough) compilers")
 @click.option("--temp-install", is_flag=True, help="Temporary install target if it's not installed yet")
+@click.option(
+    "--parallel-discovery",
+    default=4,
+    metavar="WORKERS",
+    help="Number of parallel workers for library discovery phase (default: 4, use 1 to disable)",
+)
 @click.argument("filter_", metavar="FILTER", nargs=-1)
 def build(
     context: CliContext,
@@ -766,6 +772,7 @@ def build(
     buildfor: str,
     popular_compilers_only: bool,
     temp_install: bool,
+    parallel_discovery: int,
 ):
     """Build library targets matching FILTER."""
     num_installed = 0
@@ -801,7 +808,7 @@ def build(
             else:
                 try:
                     [num_installed, num_skipped, num_failed] = installable.build(
-                        buildfor, popular_compilers_only, platform
+                        buildfor, popular_compilers_only, platform, parallel_discovery
                     )
                     if num_installed > 0:
                         _LOGGER.info("%s built OK", installable.name)

bin/lib/fortran_library_builder.py

@@ -11,6 +11,7 @@
 import shutil
 import subprocess
 import tempfile
+import threading
 import time
 from collections import defaultdict
 from collections.abc import Generator
@@ -84,6 +85,7 @@ def __init__(
         install_context,
         buildconfig: LibraryBuildConfig,
         popular_compilers_only: bool,
+        parallel_discovery_workers: int,
     ):
         self.logger = logger
         self.language = language
@@ -100,7 +102,11 @@ def __init__(
         self.conanserverproxy_token = None
         self._conan_hash_cache: dict[str, str | None] = {}
         self._annotations_cache: dict[str, dict] = {}
-        self.http_session = requests.Session()
+        self.parallel_discovery_workers = parallel_discovery_workers
+
+        # Thread-local storage for HTTP sessions
+        self._thread_local_data = threading.local()
+        self._thread_local_data.session = requests.Session()
 
         if self.language in _propsandlibs:
             [self.compilerprops, self.libraryprops] = _propsandlibs[self.language]
@@ -114,6 +120,11 @@ def __init__(
 
         self.completeBuildConfig()
 
+    @property
+    def http_session(self):
+        """Thread-local HTTP session."""
+        return self._thread_local_data.session
+
     def completeBuildConfig(self):
         if "description" in self.libraryprops[self.libid]:
             self.buildconfig.description = self.libraryprops[self.libid]["description"]

bin/lib/installable/installable.py

@@ -256,7 +256,9 @@ def sort_key(self):
     def nightly_like(self) -> bool:
         return self.install_always or self.target_name in ["nightly", "trunk", "master", "main"]
 
-    def build(self, buildfor: str, popular_compilers_only: bool, platform: LibraryPlatform):
+    def build(
+        self, buildfor: str, popular_compilers_only: bool, platform: LibraryPlatform, parallel_discovery: int = 4
+    ):
         if not self.is_library:
             raise RuntimeError("Nothing to build")
 
@@ -275,6 +277,7 @@ def build(self, buildfor: str, popular_compilers_only: bool, platform: LibraryPl
                 self.build_config,
                 popular_compilers_only,
                 platform,
+                parallel_discovery,
             )
             return cppbuilder.makebuild(buildfor)
         elif (
@@ -293,6 +296,7 @@ def build(self, buildfor: str, popular_compilers_only: bool, platform: LibraryPl
                 self.build_config,
                 popular_compilers_only,
                 platform,
+                parallel_discovery,
             )
             return cppbuilder.makebuild(buildfor)
         elif self.build_config.build_type == "fpm":
@@ -306,11 +310,18 @@ def build(self, buildfor: str, popular_compilers_only: bool, platform: LibraryPl
                 self.install_context,
                 self.build_config,
                 popular_compilers_only,
+                parallel_discovery,
             )
             return fbuilder.makebuild(buildfor)
         elif self.build_config.build_type == "cargo":
             rbuilder = RustLibraryBuilder(
-                _LOGGER, self.language, self.context[-1], self.target_name, self.install_context, self.build_config
+                _LOGGER,
+                self.language,
+                self.context[-1],
+                self.target_name,
+                self.install_context,
+                self.build_config,
+                parallel_discovery,
             )
             return rbuilder.makebuild(buildfor)
         raise RuntimeError(f"Unsupported build_type ${self.build_config.build_type}")