Allow subprojects to be built for both the build and host machine

docs/markdown/snippets/subproject_and_override_program_native.md

@@ -0,0 +1,13 @@
+## subproject() and meson.override_find_program() now support the native keyword argument
+
+Subprojects may now be built for the host or build machine (or both). This means
+that build time dependencies can be built for the matching running the build in a
+cross compile setup. When a subproject is run for the build machine it will act
+just like a normal build == host setup, except that no targets will be installed.
+
+This necessarily means that `meson.override_find_program()` must differentiate
+between programs for the host and those for the build machine, as you may need
+two versions of the same program, which have different outputs based on the
+machine they are for. For backwards compatibility reasons the default is for the
+host machine. Inside a native subproject the host and build machine will both be
+the build machine.

docs/yaml/builtins/meson.yaml

@@ -139,7 +139,7 @@ methods:
           to install only a subset of the files.
           By default the script has no install tag which means it is not being run when
           `meson install --tags` argument is specified.
-      
+
       dry_run:
         type: bool
         since: 1.1.0
@@ -392,6 +392,15 @@ methods:
         type: exe | file | external_program
         description: The program to set as the override for `progname`.
 
+      native:
+        type: bool
+        since: 1.3.0
+        default: false
+        description : |
+          If set to `true`, the override is for the build machine, and will be returned
+          by `find_program(..., native : true)`, otherwise for the host machine. This
+          is an important distinction when cross compiling.
+
   - name: override_dependency
     returns: void
     since: 0.54.0

docs/yaml/functions/subproject.yaml

@@ -65,3 +65,10 @@ kwargs:
     default: true
     description: |
       Works just the same as in [[dependency]].
+
+  native:
+    type: bool
+    since: 1.3.0
+    default: false
+    description : |
+      Whether to build this subproject for the host or build machine.

mesonbuild/ast/introspection.py

@@ -273,7 +273,7 @@ def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]:
         # Passing the unresolved sources list causes errors
         kwargs_reduced['_allow_no_sources'] = True
         target = targetclass(name, self.subdir, self.subproject, for_machine, empty_sources, None, objects,
-                             self.environment, self.coredata.compilers[for_machine], kwargs_reduced)
+                             self.environment, self.coredata.compilers[for_machine], self.coredata.is_build_only, kwargs_reduced)
         target.process_compilers_late()
 
         new_target = {

mesonbuild/backend/backends.py

@@ -321,7 +321,7 @@ def get_target_debug_filename_abs(self, target: build.BuildTarget) -> T.Optional
         return os.path.join(self.environment.get_build_dir(), self.get_target_debug_filename(target))
 
     def get_source_dir_include_args(self, target: build.BuildTarget, compiler: 'Compiler', *, absolute_path: bool = False) -> T.List[str]:
-        curdir = target.get_subdir()
+        curdir = target.get_source_subdir()
         if absolute_path:
             lead = self.source_dir
         else:
@@ -331,9 +331,9 @@ def get_source_dir_include_args(self, target: build.BuildTarget, compiler: 'Comp
 
     def get_build_dir_include_args(self, target: build.BuildTarget, compiler: 'Compiler', *, absolute_path: bool = False) -> T.List[str]:
         if absolute_path:
-            curdir = os.path.join(self.build_dir, target.get_subdir())
+            curdir = os.path.join(self.build_dir, target.get_output_subdir())
         else:
-            curdir = target.get_subdir()
+            curdir = target.get_output_subdir()
             if curdir == '':
                 curdir = '.'
         return compiler.get_include_args(curdir, False)
@@ -368,7 +368,7 @@ def get_target_dir(self, target: T.Union[build.Target, build.CustomTargetIndex])
             # this produces no output, only a dummy top-level name
             dirname = ''
         elif self.environment.coredata.get_option(OptionKey('layout')) == 'mirror':
-            dirname = target.get_subdir()
+            dirname = target.get_output_subdir()
         else:
             dirname = 'meson-out'
         return dirname
@@ -480,7 +480,7 @@ def _flatten_object_list(self, target: build.BuildTarget,
         for obj in objects:
             if isinstance(obj, str):
                 o = os.path.join(proj_dir_to_build_root,
-                                 self.build_to_src, target.get_subdir(), obj)
+                                 self.build_to_src, target.get_source_subdir(), obj)
                 obj_list.append(o)
             elif isinstance(obj, mesonlib.File):
                 if obj.is_built:
@@ -862,7 +862,7 @@ def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOr
                 gen_source = rel_src
             else:
                 gen_source = os.path.relpath(os.path.join(build_dir, rel_src),
-                                             os.path.join(self.environment.get_source_dir(), target.get_subdir()))
+                                             os.path.join(self.environment.get_source_dir(), target.get_source_subdir()))
         machine = self.environment.machines[target.for_machine]
         ret = self.canonicalize_filename(gen_source) + '.' + machine.get_object_suffix()
         if targetdir is not None:
@@ -1259,7 +1259,7 @@ def create_test_serialisation(self, tests: T.List['Test']) -> T.List[TestSeriali
                     if isinstance(d, build.BuildTarget):
                         for l in d.get_all_link_deps():
                             if isinstance(l, build.SharedLibrary):
-                                ld_lib_path.add(os.path.join(self.environment.get_build_dir(), l.get_subdir()))
+                                ld_lib_path.add(os.path.join(self.environment.get_build_dir(), l.get_output_subdir()))
                 if ld_lib_path:
                     t_env.prepend('LD_LIBRARY_PATH', list(ld_lib_path), ':')
 
@@ -1443,7 +1443,7 @@ def get_custom_target_sources(self, target: build.CustomTarget) -> T.List[str]:
         srcs: T.List[str] = []
         for i in target.get_sources():
             if isinstance(i, str):
-                fname = [os.path.join(self.build_to_src, target.subdir, i)]
+                fname = [os.path.join(self.build_to_src, target.get_source_subdir(), i)]
             elif isinstance(i, build.BuildTarget):
                 fname = [self.get_target_filename(i)]
             elif isinstance(i, (build.CustomTarget, build.CustomTargetIndex)):
@@ -1474,9 +1474,9 @@ def get_target_depend_files(self, target: T.Union[build.CustomTarget, build.Buil
                     deps.append(i.rel_to_builddir(self.build_to_src))
             else:
                 if absolute_paths:
-                    deps.append(os.path.join(self.environment.get_source_dir(), target.subdir, i))
+                    deps.append(os.path.join(self.environment.get_source_dir(), target.get_output_subdir(), i))
                 else:
-                    deps.append(os.path.join(self.build_to_src, target.subdir, i))
+                    deps.append(os.path.join(self.build_to_src, target.get_output_subdir(), i))
         return deps
 
     def get_custom_target_output_dir(self, target: T.Union[build.Target, build.CustomTargetIndex]) -> str:
@@ -1556,7 +1556,7 @@ def eval_custom_target_command(
                 if '@BUILD_ROOT@' in i:
                     i = i.replace('@BUILD_ROOT@', build_root)
                 if '@CURRENT_SOURCE_DIR@' in i:
-                    i = i.replace('@CURRENT_SOURCE_DIR@', os.path.join(source_root, target.subdir))
+                    i = i.replace('@CURRENT_SOURCE_DIR@', os.path.join(source_root, target.get_source_subdir()))
                 if '@DEPFILE@' in i:
                     if target.depfile is None:
                         msg = f'Custom target {target.name!r} has @DEPFILE@ but no depfile ' \
@@ -1605,7 +1605,7 @@ def get_run_target_env(self, target: build.RunTarget) -> mesonlib.EnvironmentVar
         if target.default_env:
             env.set('MESON_SOURCE_ROOT', [self.environment.get_source_dir()])
             env.set('MESON_BUILD_ROOT', [self.environment.get_build_dir()])
-            env.set('MESON_SUBDIR', [target.subdir])
+            env.set('MESON_SUBDIR', [target.get_source_subdir()])
             env.set('MESONINTROSPECT', [self.get_introspect_command()])
         return env
 
@@ -1942,7 +1942,7 @@ def get_introspection_data(self, target_id: str, target: build.Target) -> T.List
                 elif isinstance(j, str):
                     source_list += [os.path.join(self.source_dir, j)]
                 elif isinstance(j, (build.CustomTarget, build.BuildTarget)):
-                    source_list += [os.path.join(self.build_dir, j.get_subdir(), o) for o in j.get_outputs()]
+                    source_list += [os.path.join(self.build_dir, j.get_output_subdir(), o) for o in j.get_outputs()]
             source_list = [os.path.normpath(s) for s in source_list]
 
             compiler: T.List[str] = []

mesonbuild/backend/ninjabackend.py

@@ -1499,14 +1499,14 @@ def generate_cs_target(self, target: build.BuildTarget):
         self.generate_generator_list_rules(target)
         self.create_target_source_introspection(target, compiler, commands, rel_srcs, generated_rel_srcs)
 
-    def determine_single_java_compile_args(self, target, compiler):
+    def determine_single_java_compile_args(self, target: build.BuildTarget, compiler):
         args = []
         args += self.build.get_global_args(compiler, target.for_machine)
         args += self.build.get_project_args(compiler, target.subproject, target.for_machine)
         args += target.get_java_args()
         args += compiler.get_output_args(self.get_target_private_dir(target))
         args += target.get_classpath_args()
-        curdir = target.get_subdir()
+        curdir = target.get_source_subdir()
         sourcepath = os.path.join(self.build_to_src, curdir) + os.pathsep
         sourcepath += os.path.normpath(curdir) + os.pathsep
         for i in target.include_dirs:
@@ -1627,7 +1627,7 @@ def generate_vala_compile(self, target: build.BuildTarget) -> \
         # All sources that are passed to valac on the commandline
         all_files = list(vapi_src)
         # Passed as --basedir
-        srcbasedir = os.path.join(self.build_to_src, target.get_subdir())
+        srcbasedir = os.path.join(self.build_to_src, target.get_source_subdir())
         for (vala_file, gensrc) in vala_src.items():
             all_files.append(vala_file)
             # Figure out where the Vala compiler will write the compiled C file
@@ -1646,13 +1646,13 @@ def generate_vala_compile(self, target: build.BuildTarget) -> \
             if isinstance(gensrc, (build.CustomTarget, build.GeneratedList)) or gensrc.is_built:
                 vala_c_file = os.path.splitext(os.path.basename(vala_file))[0] + '.c'
                 # Check if the vala file is in a subdir of --basedir
-                abs_srcbasedir = os.path.join(self.environment.get_source_dir(), target.get_subdir())
+                abs_srcbasedir = os.path.join(self.environment.get_source_dir(), target.get_source_subdir())
                 abs_vala_file = os.path.join(self.environment.get_build_dir(), vala_file)
                 if PurePath(os.path.commonpath((abs_srcbasedir, abs_vala_file))) == PurePath(abs_srcbasedir):
                     vala_c_subdir = PurePath(abs_vala_file).parent.relative_to(abs_srcbasedir)
                     vala_c_file = os.path.join(str(vala_c_subdir), vala_c_file)
             else:
-                path_to_target = os.path.join(self.build_to_src, target.get_subdir())
+                path_to_target = os.path.join(self.build_to_src, target.get_source_subdir())
                 if vala_file.startswith(path_to_target):
                     vala_c_file = os.path.splitext(os.path.relpath(vala_file, path_to_target))[0] + '.c'
                 else:
@@ -1773,7 +1773,7 @@ def generate_cython_transpile(self, target: build.BuildTarget) -> \
                 if isinstance(gen, GeneratedList):
                     ssrc = os.path.join(self.get_target_private_dir(target), ssrc)
                 else:
-                    ssrc = os.path.join(gen.get_subdir(), ssrc)
+                    ssrc = os.path.join(gen.get_output_subdir(), ssrc)
                 if ssrc.endswith('.pyx'):
                     output = os.path.join(self.get_target_private_dir(target), f'{ssrc}.{ext}')
                     element = NinjaBuildElement(
@@ -1786,7 +1786,7 @@ def generate_cython_transpile(self, target: build.BuildTarget) -> \
                     # TODO: introspection?
                     cython_sources.append(output)
                 else:
-                    generated_sources[ssrc] = mesonlib.File.from_built_file(gen.get_subdir(), ssrc)
+                    generated_sources[ssrc] = mesonlib.File.from_built_file(gen.get_output_subdir(), ssrc)
                     # Following logic in L883-900 where we determine whether to add generated source
                     # as a header(order-only) dep to the .so compilation rule
                     if not self.environment.is_source(ssrc) and \
@@ -1899,7 +1899,7 @@ def generate_rust_target(self, target: build.BuildTarget) -> None:
                 elif isinstance(g, GeneratedList):
                     main_rust_file = os.path.join(self.get_target_private_dir(target), g.get_outputs()[0])
                 else:
-                    main_rust_file = os.path.join(g.get_subdir(), g.get_outputs()[0])
+                    main_rust_file = os.path.join(g.get_output_subdir(), g.get_outputs()[0])
 
                 for f in target.structured_sources.as_list():
                     if isinstance(f, File):
@@ -1920,13 +1920,13 @@ def generate_rust_target(self, target: build.BuildTarget) -> None:
                 if isinstance(g, GeneratedList):
                     fname = os.path.join(self.get_target_private_dir(target), i)
                 else:
-                    fname = os.path.join(g.get_subdir(), i)
+                    fname = os.path.join(g.get_output_subdir(), i)
                 if main_rust_file is None:
                     main_rust_file = fname
                 orderdeps.append(fname)
         if main_rust_file is None:
             raise RuntimeError('A Rust target has no Rust sources. This is weird. Also a bug. Please report')
-        target_name = os.path.join(target.subdir, target.get_filename())
+        target_name = os.path.join(target.get_output_subdir(), target.get_filename())
         cratetype = target.rust_crate_type
         args.extend(['--crate-type', cratetype])
 
@@ -2784,28 +2784,6 @@ def generate_llvm_ir_compile(self, target, src):
         self.add_build(element)
         return (rel_obj, rel_src)
 
-    @lru_cache(maxsize=None)
-    def generate_inc_dir(self, compiler: 'Compiler', d: str, basedir: str, is_system: bool) -> \
-            T.Tuple['ImmutableListProtocol[str]', 'ImmutableListProtocol[str]']:
-        # Avoid superfluous '/.' at the end of paths when d is '.'
-        if d not in ('', '.'):
-            expdir = os.path.normpath(os.path.join(basedir, d))
-        else:
-            expdir = basedir
-        srctreedir = os.path.normpath(os.path.join(self.build_to_src, expdir))
-        sargs = compiler.get_include_args(srctreedir, is_system)
-        # There may be include dirs where a build directory has not been
-        # created for some source dir. For example if someone does this:
-        #
-        # inc = include_directories('foo/bar/baz')
-        #
-        # But never subdir()s into the actual dir.
-        if os.path.isdir(os.path.join(self.environment.get_build_dir(), expdir)):
-            bargs = compiler.get_include_args(expdir, is_system)
-        else:
-            bargs = []
-        return (sargs, bargs)
-
     def _generate_single_compile(self, target: build.BuildTarget, compiler: Compiler) -> CompilerArgs:
         commands = self._generate_single_compile_base_args(target, compiler)
         commands += self._generate_single_compile_target_args(target, compiler)
@@ -2844,16 +2822,16 @@ def _generate_single_compile_target_args(self, target: build.BuildTarget, compil
         # external deps and must maintain the order in which they are specified.
         # Hence, we must reverse the list so that the order is preserved.
         for i in reversed(target.get_include_dirs()):
-            basedir = i.get_curdir()
             # We should iterate include dirs in reversed orders because
             # -Ipath will add to begin of array. And without reverse
             # flags will be added in reversed order.
-            for d in reversed(i.get_incdirs()):
+            for d in reversed(i.expand_incdirs(self.environment.get_build_dir())):
                 # Add source subdir first so that the build subdir overrides it
-                (compile_obj, includeargs) = self.generate_inc_dir(compiler, d, basedir, i.is_system)
-                commands += compile_obj
-                commands += includeargs
-            for d in i.get_extra_build_dirs():
+                commands += compiler.get_include_args(os.path.normpath(os.path.join(self.build_to_src, d.source)),
+                                                      i.is_system)
+                if d.build is not None:
+                    commands += compiler.get_include_args(d.build, i.is_system)
+            for d in i.expand_extra_build_dirs():
                 commands += compiler.get_include_args(d, i.is_system)
         # Add per-target compile args, f.ex, `c_args : ['-DFOO']`. We set these
         # near the end since these are supposed to override everything else.

mesonbuild/backend/vs2010backend.py

@@ -1023,18 +1023,17 @@ def get_args_defines_and_inc_dirs(self, target, compiler, generated_files_includ
             # need them to be looked in first.
             for d in reversed(target.get_include_dirs()):
                 # reversed is used to keep order of includes
-                for i in reversed(d.get_incdirs()):
-                    curdir = os.path.join(d.get_curdir(), i)
+                for i in reversed(d.expand_incdirs(self.environment.get_build_dir())):
                     try:
                         # Add source subdir first so that the build subdir overrides it
-                        args.append('-I' + os.path.join(proj_to_src_root, curdir))  # src dir
-                        args.append('-I' + self.relpath(curdir, target.subdir)) # build dir
+                        args.append('-I' + os.path.join(proj_to_src_root, i.source))
+                        if i.build is not None:
+                            args.append('-I' + self.relpath(i.build, target.subdir))
                     except ValueError:
                         # Include is on different drive
-                        args.append('-I' + os.path.normpath(curdir))
-                for i in d.get_extra_build_dirs():
-                    curdir = os.path.join(d.get_curdir(), i)
-                    args.append('-I' + self.relpath(curdir, target.subdir))  # build dir
+                        args.append('-I' + os.path.normpath(i.build))
+                for i in d.expand_extra_build_dirs():
+                    args.append('-I' + self.relpath(i, target.subdir))
         # Add per-target compile args, f.ex, `c_args : ['/DFOO']`. We set these
         # near the end since these are supposed to override everything else.
         for l, args in target.extra_args.items():
@@ -1724,7 +1723,7 @@ def path_normalize_add(path, lis):
                         self.add_additional_options(lang, inc_cl, file_args)
                         self.add_preprocessor_defines(lang, inc_cl, file_defines)
                         self.add_include_dirs(lang, inc_cl, file_inc_dirs)
-                        s = File.from_built_file(target.get_subdir(), s)
+                        s = File.from_built_file(target.get_output_subdir(), s)
                         ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + \
                             self.object_filename_from_source(target, s)
             for lang, headers in pch_sources.items():