intrinsic-test: simplify linker logic

just always compile .cpp to .o to maximize thread utilization, and only invoke the (maybe custom) linker at the very end

Author: folkertdev

crates/intrinsic-test/src/arm/compile.rs

@@ -1,7 +1,7 @@
 use crate::common::cli::ProcessedCli;
 use crate::common::compile_c::{CompilationCommandBuilder, CppCompilation};
 
-pub fn build_cpp_compilation(config: &ProcessedCli) -> Option<CppCompilation> {
+pub fn configure_cpp_compiler(config: &ProcessedCli) -> Option<CppCompilation> {
     let cpp_compiler = config.cpp_compiler.as_ref()?;
 
     // -ffp-contract=off emulates Rust's approach of not fusing separate mul-add operations

crates/intrinsic-test/src/arm/mod.rs

@@ -59,50 +59,81 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
         let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
 
         let available_parallelism = std::thread::available_parallelism().unwrap().get();
-        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
+        let chunk_count = Ord::min(available_parallelism - 1, self.intrinsics.len());
+        let chunk_size = self.intrinsics.len().div_ceil(chunk_count);
+
+        let cpp_compiler = compile::configure_cpp_compiler(&self.cli_options).unwrap();
 
-        let pipeline = compile::build_cpp_compilation(&self.cli_options).unwrap();
+        enum Job<'a> {
+            Chunk(&'a [Intrinsic<ArmIntrinsicType>]),
+            Main,
+        }
 
         let notice = &build_notices("// ");
         self.intrinsics
             .par_chunks(chunk_size)
+            .map(Job::Chunk)
+            .chain([Job::Main])
             .enumerate()
-            .map(|(i, chunk)| {
-                let c_filename = format!("c_programs/mod_{i}.cpp");
-                let mut file = File::create(&c_filename).unwrap();
-                write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
-
-                // compile this cpp file into a .o file
-                let output = pipeline.run(&[], &[format!("mod_{i}.cpp")], &format!("mod_{i}.o"))?;
-                assert!(output.status.success());
-
-                Ok(())
+            .map(|(i, job)| {
+                match job {
+                    Job::Chunk(chunk) => {
+                        let c_filename = format!("c_programs/mod_{i}.cpp");
+                        info!("writing {c_filename}");
+                        let mut file = File::create(&c_filename).unwrap();
+                        write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk)
+                            .unwrap();
+
+                        // compile this cpp file into a .o file
+                        info!("compiling {c_filename}");
+                        let output = cpp_compiler
+                            .compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
+                        assert!(output.status.success());
+
+                        Ok(())
+                    }
+                    Job::Main => {
+                        info!("writing main.cpp");
+                        let mut file = File::create("c_programs/main.cpp").unwrap();
+                        write_main_cpp(
+                            &mut file,
+                            c_target,
+                            POLY128_OSTREAM_DEF,
+                            self.intrinsics.iter().map(|i| i.name.as_str()),
+                        )?;
+
+                        // compile this cpp file into a .o file
+                        info!("compiling main.cpp");
+                        let output = cpp_compiler
+                            .compile_object_file("main.cpp", "intrinsic-test-programs.o")?;
+                        assert!(output.status.success());
+
+                        Ok(())
+                    }
+                }
             })
             .collect::<Result<(), std::io::Error>>()
             .unwrap();
 
-        let mut file = File::create("c_programs/main.cpp").unwrap();
-        write_main_cpp(
-            &mut file,
-            c_target,
-            POLY128_OSTREAM_DEF,
-            self.intrinsics.iter().map(|i| i.name.as_str()),
-        )
-        .unwrap();
+        let mut linker = match &self.cli_options.linker {
+            Some(custom) => std::process::Command::new(custom),
+            None => {
+                let compiler = self.cli_options.cpp_compiler.as_ref().unwrap();
+                let mut cmd = std::process::Command::new(compiler);
+                cmd.args(["-target", &self.cli_options.target]);
+                cmd
+            }
+        };
 
-        // Files to include in the final link step.
-        let mut includes = vec![];
-        for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) {
-            includes.push(format!("mod_{i}.o"));
-        }
+        linker.current_dir("c_programs");
 
-        let output = pipeline
-            .run(
-                &includes,
-                &["main.cpp".to_string()],
-                "intrinsic-test-programs",
-            )
-            .unwrap();
+        linker.args((0..chunk_count).map(|i| format!("mod_{i}.o")));
+        linker.arg("intrinsic-test-programs.o");
+        linker.args(["-o", "intrinsic-test-programs"]);
+
+        let linker_name = linker.get_program().display();
+        info!("linking final C binary with {linker_name}");
+        let output = linker.output().unwrap();
         assert!(output.status.success());
 
         true

crates/intrinsic-test/src/common/compile_c.rs

@@ -104,30 +104,19 @@ impl CompilationCommandBuilder {
             cpp_compiler.arg(format!("--target={target}"));
         }
 
-        if let (Some(linker), Some(cxx_toolchain_dir)) = (&self.linker, &self.cxx_toolchain_dir) {
+        if let Some(cxx_toolchain_dir) = &self.cxx_toolchain_dir {
             cpp_compiler.args(
                 self.include_paths
                     .iter()
-                    .map(|path| "--include-directory=".to_string() + cxx_toolchain_dir + path),
+                    .map(|path| format!("--include-directory={cxx_toolchain_dir}{path}")),
             );
-
-            CppCompilation::CustomLinker {
-                cpp_compiler,
-                linker: linker.to_owned(),
-            }
-        } else {
-            CppCompilation::Simple(cpp_compiler)
         }
+
+        CppCompilation(cpp_compiler)
     }
 }
 
-pub enum CppCompilation {
-    Simple(std::process::Command),
-    CustomLinker {
-        cpp_compiler: std::process::Command,
-        linker: String,
-    },
-}
+pub struct CppCompilation(std::process::Command);
 
 fn clone_command(command: &std::process::Command) -> std::process::Command {
     let mut cmd = std::process::Command::new(command.get_program());
@@ -144,85 +133,13 @@ fn clone_command(command: &std::process::Command) -> std::process::Command {
 }
 
 impl CppCompilation {
-    fn compile_cpp(
-        command: &std::process::Command,
-        includes: &[String],
-        inputs: &[String],
-        output: &str,
-    ) -> std::io::Result<std::process::Output> {
-        let mut cmd = clone_command(command);
-        cmd.args(includes);
-        cmd.args(inputs);
-        cmd.args(["-o", output]);
-
-        if output.ends_with(".o") {
-            cmd.arg("-c");
-        }
-
-        cmd.output()
-    }
-
-    pub fn run(
+    pub fn compile_object_file(
         &self,
-        includes: &[String],
-        inputs: &[String],
+        input: &str,
         output: &str,
     ) -> std::io::Result<std::process::Output> {
-        match self {
-            CppCompilation::Simple(command) => Self::compile_cpp(command, includes, inputs, output),
-            CppCompilation::CustomLinker { cpp_compiler, .. } if output.ends_with(".o") => {
-                // No need to invoke that custom linker if we're creating an object file.
-                Self::compile_cpp(cpp_compiler, includes, inputs, output)
-            }
-            CppCompilation::CustomLinker {
-                cpp_compiler,
-                linker,
-            } => {
-                let object_file = &format!("{output}.o");
-
-                // Build an object file using the cpp compiler.
-                let mut cmd = clone_command(cpp_compiler);
-                cmd.args(inputs);
-                cmd.args(["-c", "-o", object_file]);
-
-                let cpp_output = cmd.output()?;
-                if !cpp_output.status.success() {
-                    error!("c++ compilaton failed");
-                    return Ok(cpp_output);
-                }
-
-                trace!("using custom linker");
-
-                // Use the custom linker to turn the object file into an executable.
-                let mut cmd = std::process::Command::new(linker);
-                cmd.args(includes);
-                cmd.args([object_file, "-o", output]);
-
-                if let Some(current_dir) = cpp_compiler.get_current_dir() {
-                    cmd.current_dir(current_dir);
-                }
-
-                for (key, val) in cpp_compiler.get_envs() {
-                    cmd.env(key, val.unwrap_or_default());
-                }
-
-                let linker_output = cmd.output()?;
-                if !linker_output.status.success() {
-                    error!("custom linker failed");
-                    error!("{}", String::from_utf8_lossy(&linker_output.stderr));
-                    return Ok(linker_output);
-                }
-
-                trace!("removing {object_file}");
-                let object_file_path = match cpp_compiler.get_current_dir() {
-                    Some(current_dir) => &format!("{}/{object_file}", current_dir.display()),
-                    None => object_file,
-                };
-
-                std::fs::remove_file(object_file_path)?;
-
-                Ok(cpp_output)
-            }
-        }
+        let mut cmd = clone_command(&self.0);
+        cmd.args([input, "-c", "-o", output]);
+        cmd.output()
     }
 }