Add target_toolchain_roots and make sysroot more hermetic

It is useful when cross compiling to have 1 LLVM build used as the
compiler, and multiple others used for their libc++/target libraries.

This concept forces us to be much more robust about what comes from the
target toolchain, and what comes from the compiler.  Leave
toolchain_roots for the compiler roots, but add a new flag,
target_toolchain_roots with the same semantics for target toolchains.
This necesitates being much more clear internally about what comes from
what.

This isn't fully enough to make everything hermetic, clang still has
some autodetection logic left over when using sysroots.  We want to very
explicity specify the resource directory it should use, as well as where
the standard C/C++ libraries come from so it finds only what we want it
to find.

When pulling from a sysroot, we need to know the version of stdc++ to
find includes.  Use the stdlib variable for that by supporting
"stdc++-12" for example, and parsing the number out.

Author: AustinSchuh

toolchain/BUILD.llvm_repo

@@ -48,7 +48,7 @@ filegroup(
 filegroup(
     name = "include",
     srcs = glob([
-        "include/**/c++/**",
+        "include/**",
         "lib/clang/*/include/**",
     ]),
 )
@@ -70,8 +70,11 @@ filegroup(
     name = "lib",
     srcs = glob(
         [
+            "lib/lib*.a",
             "lib/**/lib*.a",
+            "lib/**/lib*.a.syms",
             "lib/clang/*/lib/**/*.a",
+            "lib/clang/*/lib/**/*.a.syms",
             "lib/clang/*/lib/**/*.dylib",
             # clang_rt.*.o supply crtbegin and crtend sections.
             "lib/**/clang_rt.*.o",

toolchain/cc_toolchain_config.bzl

@@ -37,11 +37,12 @@ def cc_toolchain_config(
         target_os,
         target_system_name,
         toolchain_path_prefix,
+        target_toolchain_path_prefix,
         tools_path_prefix,
         wrapper_bin_prefix,
         compiler_configuration,
         cxx_builtin_include_directories,
-        major_llvm_version):
+        llvm_version):
     exec_os_arch_key = _os_arch_pair(exec_os, exec_arch)
     target_os_arch_key = _os_arch_pair(target_os, target_arch)
     _check_os_arch_keys([exec_os_arch_key, target_os_arch_key])
@@ -56,6 +57,7 @@ def cc_toolchain_config(
         compiler,
         abi_version,
         abi_libc_version,
+        multiarch,
     ) = {
         "darwin-x86_64": (
             "clang-x86_64-darwin",
@@ -64,6 +66,7 @@ def cc_toolchain_config(
             "clang",
             "darwin_x86_64",
             "darwin_x86_64",
+            None,
         ),
         "darwin-aarch64": (
             "clang-aarch64-darwin",
@@ -72,6 +75,7 @@ def cc_toolchain_config(
             "clang",
             "darwin_aarch64",
             "darwin_aarch64",
+            None,
         ),
         "linux-aarch64": (
             "clang-aarch64-linux",
@@ -80,6 +84,7 @@ def cc_toolchain_config(
             "clang",
             "clang",
             "glibc_unknown",
+            "aarch64-linux-gnu",
         ),
         "linux-x86_64": (
             "clang-x86_64-linux",
@@ -88,6 +93,7 @@ def cc_toolchain_config(
             "clang",
             "clang",
             "glibc_unknown",
+            "x86_64-linux-gnu",
         ),
         "wasm32": (
             "clang-wasm32",
@@ -96,6 +102,7 @@ def cc_toolchain_config(
             "clang",
             "unknown",
             "unknown",
+            None,
         ),
         "wasm64": (
             "clang-wasm64",
@@ -104,6 +111,7 @@ def cc_toolchain_config(
             "clang",
             "unknown",
             "unknown",
+            None,
         ),
     }[target_os_arch_key]
 
@@ -120,12 +128,30 @@ def cc_toolchain_config(
         "-D__TIME__=\"redacted\"",
         "-fdebug-prefix-map={}=__bazel_toolchain_llvm_repo__/".format(toolchain_path_prefix),
     ]
+    if target_toolchain_path_prefix != toolchain_path_prefix:
+        unfiltered_compile_flags.extend([
+            "-fdebug-prefix-map={}=__bazel_toolchain_llvm_repo__/".format(target_toolchain_path_prefix),
+        ])
 
     is_xcompile = not (exec_os == target_os and exec_arch == target_arch)
 
-    # Default compiler flags:
-    compile_flags = [
+    major_llvm_version = int(llvm_version.split(".")[0])
+
+    external_include_paths = None
+
+    resource_dir_version = llvm_version if major_llvm_version < 16 else major_llvm_version
+
+    resource_dir = [
+        "-resource-dir",
+        "{}lib/clang/{}".format(target_toolchain_path_prefix, resource_dir_version),
+    ]
+
+    target_flags = [
         "--target=" + target_system_name,
+    ]
+
+    # Default compiler flags:
+    compile_flags = target_flags + [
         # Security
         "-U_FORTIFY_SOURCE",  # https://github.com/google/sanitizers/issues/247
         "-fstack-protector",
@@ -135,22 +161,36 @@ def cc_toolchain_config(
         "-Wall",
         "-Wthread-safety",
         "-Wself-assign",
-    ]
+        "-B{}bin/".format(toolchain_path_prefix),
+    ] + resource_dir
 
     dbg_compile_flags = ["-g", "-fstandalone-debug"]
 
     opt_compile_flags = [
-        "-g0",
+        # Disable debug info
+        "-ggdb0",
         "-O2",
         "-D_FORTIFY_SOURCE=1",
         "-DNDEBUG",
         "-ffunction-sections",
         "-fdata-sections",
     ]
 
-    link_flags = [
-        "--target=" + target_system_name,
+    link_flags = target_flags + [
         "-no-canonical-prefixes",
+    ] + resource_dir
+
+    # Use the standard libc++ location when not using msan
+    non_msan_compile_flags = [
+        "-isystem",
+        target_toolchain_path_prefix + "include/c++/v1/",
+        "-isystem",
+        target_toolchain_path_prefix + "include/" + target_system_name + "/c++/v1/",
+    ]
+
+    non_msan_link_flags = [
+        "-L{}lib".format(target_toolchain_path_prefix),
+        "-L{}lib/{}".format(target_toolchain_path_prefix, target_system_name),
     ]
 
     stdlib = compiler_configuration["stdlib"]
@@ -210,13 +250,30 @@ def cc_toolchain_config(
     cxx_standard = compiler_configuration["cxx_standard"]
     conly_flags = compiler_configuration["conly_flags"]
     sysroot_path = compiler_configuration["sysroot_path"]
+
+    # Flags related to C++ standard.
+    cxx_flags = [
+        "-std=" + cxx_standard,
+    ]
+    compile_flags_post_language = []
+
+    # We only support getting libc++ from the toolchain for now. Is it worth
+    # supporting libc++ from the sysroot? Or maybe just part of a LLVM distribution
+    # that's built for the target?
+    if not stdlib and is_xcompile:
+        print("WARNING: Using libc++ for host architecture while cross compiling, this is " +
+              "probably not what you want. Explicitly set standard_libraries to libc++ to silence.")
+
     if stdlib == "builtin-libc++" and is_xcompile:
         stdlib = "stdc++"
     if stdlib == "builtin-libc++":
-        cxx_flags = [
-            "-std=" + cxx_standard,
+        cxx_flags.extend([
             "-stdlib=libc++",
-        ]
+        ])
+        compile_flags_post_language.extend([
+            "-isystem",
+            target_toolchain_path_prefix + "lib/clang/{}/include".format(resource_dir_version),
+        ])
         if major_llvm_version >= 14:
             # With C++20, Clang defaults to using C++ rather than Clang modules,
             # which breaks Bazel's `use_module_maps` feature, which is used by
@@ -260,37 +317,80 @@ def cc_toolchain_config(
             ]
 
     elif stdlib == "libc++":
-        cxx_flags = [
-            "-std=" + cxx_standard,
+        cxx_flags.extend([
             "-stdlib=libc++",
-        ]
+        ])
 
         link_flags.extend([
             "-l:c++.a",
             "-l:c++abi.a",
         ])
     elif stdlib == "dynamic-stdc++":
-        cxx_flags = [
+        cxx_flags.extend([
             "-std=" + cxx_standard,
             "-stdlib=libstdc++",
-        ]
+        ])
 
         link_flags.extend([
             "-lstdc++",
         ])
-    elif stdlib == "stdc++":
-        cxx_flags = [
-            "-std=" + cxx_standard,
-            "-stdlib=libstdc++",
-        ]
+    elif stdlib.startswith("stdc++"):
+        if not use_lld:
+            fail("libstdc++ only supported with lld")
 
+        # We use libgcc when using libstdc++ from a sysroot. Most libstdc++
+        # builds link to libgcc, which means we need to use libgcc's exception
+        # handling implementation, not the separate one in compiler-rt.
+        # Unfortunately, clang sometimes emits code incompatible with libgcc,
+        # see <https://bugs.llvm.org/show_bug.cgi?id=27455> and
+        # <https://lists.llvm.org/pipermail/cfe-dev/2016-April/048466.html> for
+        # example. This seems to be a commonly-used configuration with clang
+        # though, so it's probably good enough for most people.
+        link_flags.extend([
+            "-L{}lib".format(target_toolchain_path_prefix),
+            "-L{}lib/{}".format(target_toolchain_path_prefix, target_system_name),
+        ])
+
+        # We expect to pick up these libraries from the sysroot.
         link_flags.extend([
             "-l:libstdc++.a",
         ])
+        if stdlib == "stdc++":
+            cxx_flags.extend([
+                "-stdlib=libstdc++",
+            ])
+        elif stdlib.startswith("stdc++-"):
+            if sysroot_path == None:
+                fail("Need a sysroot to link against stdc++")
+
+            # -stdlib does nothing when using -nostdinc besides produce a warning
+            # that it's unused, so don't use it here.
+            libstdcxx_version = stdlib[len("stdc++-"):]
+
+            cxx_flags.extend([
+                "-nostdinc++",
+                "-isystem",
+                sysroot_path + "/usr/include/c++/" + libstdcxx_version,
+                "-isystem",
+                sysroot_path + "/usr/include/" + multiarch + "/c++/" + libstdcxx_version,
+                "-isystem",
+                sysroot_path + "/usr/include/c++/" + libstdcxx_version + "/backward",
+            ])
+
+            # Clang really wants C system header includes after C++ ones.
+            compile_flags_post_language.extend([
+                "-nostdinc",
+                "-isystem",
+                target_toolchain_path_prefix + "lib/clang/{}/include".format(resource_dir_version),
+            ])
+
+            link_flags.extend([
+                "-L" + sysroot_path + "/usr/lib/gcc/" + multiarch + "/" + libstdcxx_version,
+            ])
+        else:
+            fail("Invalid stdlib: " + stdlib)
     elif stdlib == "libc":
-        cxx_flags = [
-            "-std=" + cxx_standard,
-        ]
+        pass
     elif stdlib == "none":
         cxx_flags = [
             "-nostdlib",
@@ -343,27 +443,46 @@ def cc_toolchain_config(
 
     # Replace flags with any user-provided overrides.
     if compiler_configuration["compile_flags"] != None:
-        compile_flags = _fmt_flags(compiler_configuration["compile_flags"], toolchain_path_prefix)
+        compile_flags.extend(_fmt_flags(compiler_configuration["compile_flags"], toolchain_path_prefix))
     if compiler_configuration["cxx_flags"] != None:
-        cxx_flags = _fmt_flags(compiler_configuration["cxx_flags"], toolchain_path_prefix)
+        cxx_flags.extend(_fmt_flags(compiler_configuration["cxx_flags"], toolchain_path_prefix))
     if compiler_configuration["link_flags"] != None:
-        link_flags = _fmt_flags(compiler_configuration["link_flags"], toolchain_path_prefix)
+        link_flags.extend(_fmt_flags(compiler_configuration["link_flags"], toolchain_path_prefix))
     if compiler_configuration["archive_flags"] != None:
-        archive_flags = _fmt_flags(compiler_configuration["archive_flags"], toolchain_path_prefix)
+        archive_flags.extend(_fmt_flags(compiler_configuration["archive_flags"], toolchain_path_prefix))
     if compiler_configuration["link_libs"] != None:
-        link_libs = _fmt_flags(compiler_configuration["link_libs"], toolchain_path_prefix)
+        link_libs.extend(_fmt_flags(compiler_configuration["link_libs"], toolchain_path_prefix))
     if compiler_configuration["opt_compile_flags"] != None:
-        opt_compile_flags = _fmt_flags(compiler_configuration["opt_compile_flags"], toolchain_path_prefix)
+        opt_compile_flags.extend(_fmt_flags(compiler_configuration["opt_compile_flags"], toolchain_path_prefix))
     if compiler_configuration["opt_link_flags"] != None:
-        opt_link_flags = _fmt_flags(compiler_configuration["opt_link_flags"], toolchain_path_prefix)
+        opt_link_flags.extend(_fmt_flags(compiler_configuration["opt_link_flags"], toolchain_path_prefix))
     if compiler_configuration["dbg_compile_flags"] != None:
-        dbg_compile_flags = _fmt_flags(compiler_configuration["dbg_compile_flags"], toolchain_path_prefix)
+        dbg_compile_flags.extend(_fmt_flags(compiler_configuration["dbg_compile_flags"], toolchain_path_prefix))
     if compiler_configuration["coverage_compile_flags"] != None:
-        coverage_compile_flags = _fmt_flags(compiler_configuration["coverage_compile_flags"], toolchain_path_prefix)
+        coverage_compile_flags.extend(_fmt_flags(compiler_configuration["coverage_compile_flags"], toolchain_path_prefix))
     if compiler_configuration["coverage_link_flags"] != None:
-        coverage_link_flags = _fmt_flags(compiler_configuration["coverage_link_flags"], toolchain_path_prefix)
+        coverage_link_flags.extend(_fmt_flags(compiler_configuration["coverage_link_flags"], toolchain_path_prefix))
     if compiler_configuration["unfiltered_compile_flags"] != None:
-        unfiltered_compile_flags = _fmt_flags(compiler_configuration["unfiltered_compile_flags"], toolchain_path_prefix)
+        unfiltered_compile_flags.extend(_fmt_flags(compiler_configuration["unfiltered_compile_flags"], toolchain_path_prefix))
+
+    # TODO(AustinSchuh): Add msan support and make this conditional.
+    cxx_flags = non_msan_compile_flags + cxx_flags
+
+    # Add the sysroot flags here, as we want to check these last
+    if sysroot_path != None:
+        external_include_paths = [
+            sysroot_path + "usr/local/include",
+        ]
+        if multiarch != None:
+            external_include_paths.extend([
+                sysroot_path + "usr/" + multiarch + "/include",
+                sysroot_path + "usr/include/" + multiarch,
+            ])
+
+        external_include_paths.extend([
+            sysroot_path + "usr/include",
+            sysroot_path + "include",
+        ])
 
     # Source: https://cs.opensource.google/bazel/bazel/+/master:tools/cpp/unix_cc_toolchain_config.bzl
     unix_cc_toolchain_config(
@@ -383,8 +502,10 @@ def cc_toolchain_config(
         opt_compile_flags = opt_compile_flags,
         conly_flags = conly_flags,
         cxx_flags = cxx_flags,
+        compile_flags_post_language = compile_flags_post_language,
         link_flags = link_flags + select({str(Label("@toolchains_llvm//toolchain/config:use_libunwind")): libunwind_link_flags, "//conditions:default": []}) +
-                     select({str(Label("@toolchains_llvm//toolchain/config:use_compiler_rt")): compiler_rt_link_flags, "//conditions:default": []}),
+                     select({str(Label("@toolchains_llvm//toolchain/config:use_compiler_rt")): compiler_rt_link_flags, "//conditions:default": []}) +
+                     non_msan_link_flags,
         archive_flags = archive_flags,
         link_libs = link_libs,
         opt_link_flags = opt_link_flags,