Enable formatting of environment variables in cc_args

Copybara Import from #413

BEGIN_PUBLIC
Enable formatting of environment variables in cc_args (#413)

Windows does not follow the Unix sysroot convention.
Instead, it uses the `INCLUDE` and `LIB` environment variables
to specify the include and library search paths.

This change enables formatting of the values of envirnoment variables set
in the `env` attribute of `cc_args` using the standard `format` attribute.
This allows Windows users to download a hermetic "sysroot" as part of the build,
and set the relevant "INCLUDE" and "LIB" environment variables.

Example usage:
```
cc_args(
    name = "arg-include",
    actions = [
        "@rules_cc//cc/toolchains/actions:c_compile",
        "@rules_cc//cc/toolchains/actions:cpp_compile_actions",
    ],
    env = {
        "INCLUDE": "{include}",
    },
    format = {
        "include": ":sysroot-include",
    },
    data = [
        ":sysroot-include",
    ],
)
```

example repo: https://github.com/reutermj/windows-rules-based-toolchain-example

Future work:
Current implementation for the format substitution only allows for a single variable to be substituted. Users of the API often will need to substitute in multiple paths into the `INCLUDE`/`LIB` environment variables.

Closes #413
END_PUBLIC

COPYBARA_INTEGRATE_REVIEW=#413 from reutermj:main 4c59d66
PiperOrigin-RevId: 780817009
Change-Id: Ifab4889f94f788f24a2e585d124444b49102839b

Author: reutermj

cc/toolchains/args.bzl

@@ -24,6 +24,7 @@ load(
 load(
     "//cc/toolchains/impl:nested_args.bzl",
     "NESTED_ARGS_ATTRS",
+    "format_dict_values",
     "nested_args_provider_from_ctx",
 )
 load(
@@ -40,9 +41,17 @@ visibility("public")
 def _cc_args_impl(ctx):
     actions = collect_action_types(ctx.attr.actions)
 
+    formatted_env, used_format_vars = format_dict_values(
+        env = ctx.attr.env,
+        must_use = [],  # checking for unused variables in done when formatting `args`.
+        format = {k: v for v, k in ctx.attr.format.items()},
+    )
+
     nested = None
     if ctx.attr.args or ctx.attr.nested:
-        nested = nested_args_provider_from_ctx(ctx)
+        # Forward the format variables used by the env formatting so they don't trigger
+        # errors if they go unused during the argument formatting.
+        nested = nested_args_provider_from_ctx(ctx, used_format_vars)
         validate_nested_args(
             variables = ctx.attr._variables[BuiltinVariablesInfo].variables,
             nested_args = nested,
@@ -60,7 +69,7 @@ def _cc_args_impl(ctx):
         actions = actions,
         requires_any_of = tuple(requires),
         nested = nested,
-        env = ctx.attr.env,
+        env = formatted_env,
         files = files,
         allowlist_include_directories = depset(
             direct = [d[DirectoryInfo] for d in ctx.attr.allowlist_include_directories],
@@ -234,10 +243,11 @@ def cc_args(
         data: (List[Label]) A list of runtime data dependencies that are required for these
             arguments to work as intended.
         env: (Dict[str, str]) Environment variables that should be set when the tool is invoked.
-        format: (Dict[str, Label]) A mapping of format strings to the label of the corresponding
-            `cc_variable` that the value should be pulled from. All instances of
-            `{variable_name}` will be replaced with the expanded value of `variable_name` in this
-            dictionary. The complete list of possible variables can be found in
+        format: (Dict[str, Label]) A mapping of format strings to the label of a corresponding
+            target. This target can be a `directory`, `subdirectory`, `cc_variable`, or a single
+            file that the value should be pulled from. All instances of `{variable_name}` in the
+            `args` list will be replaced with the expanded value in this dictionary.
+            The complete list of possible variables can be found in
             https://github.com/bazelbuild/rules_cc/tree/main/cc/toolchains/variables/BUILD.
             It is not possible to declare custom variables--these are inherent to Bazel itself.
         iterate_over: (Label) The label of a `cc_variable` that should be iterated over. This is

cc/toolchains/impl/nested_args.bzl

@@ -84,11 +84,13 @@ def _var(target):
 
 # TODO: Consider replacing this with a subrule in the future. However, maybe not
 # for a long time, since it'll break compatibility with all bazel versions < 7.
-def nested_args_provider_from_ctx(ctx):
+def nested_args_provider_from_ctx(ctx, maybe_used_vars = []):
     """Gets the nested args provider from a rule that has NESTED_ARGS_ATTRS.
 
     Args:
         ctx: The rule context
+        maybe_used_vars: (List[str]) A list of format variables that are not needed during args formatting.
+
     Returns:
         NestedArgsInfo
     """
@@ -105,6 +107,7 @@ def nested_args_provider_from_ctx(ctx):
         requires_false = _var(ctx.attr.requires_false),
         requires_equal = _var(ctx.attr.requires_equal),
         requires_equal_value = ctx.attr.requires_equal_value,
+        maybe_used_vars = maybe_used_vars,
     )
 
 def nested_args_provider(
@@ -121,6 +124,7 @@ def nested_args_provider(
         requires_false = None,
         requires_equal = None,
         requires_equal_value = "",
+        maybe_used_vars = [],
         fail = fail):
     """Creates a validated NestedArgsInfo.
 
@@ -148,6 +152,7 @@ def nested_args_provider(
           be ignored if the variable is not equal to requires_equal_value.
         requires_equal_value: (str) The value to compare the requires_equal
           variable with
+        maybe_used_vars: (List[str]) A list of format variables that are not needed during args formatting.
         fail: A fail function. Use only for testing.
     Returns:
         NestedArgsInfo
@@ -183,7 +188,12 @@ def nested_args_provider(
     #     args = ["{}"],
     #     iterate_over = "//cc/toolchains/variables:libraries_to_link.object_files",
     # )
-    args = format_args(args, replacements, must_use = format.values(), fail = fail)
+    formatted_args, _ = format_list(
+        args,
+        replacements,
+        must_use = [var for var in format.values() if var not in maybe_used_vars],
+        fail = fail,
+    )
 
     transitive_files = [ea.files for ea in nested]
     transitive_files.append(files)
@@ -203,8 +213,8 @@ def nested_args_provider(
 
     kwargs = {}
 
-    if args:
-        kwargs["flags"] = args
+    if formatted_args:
+        kwargs["flags"] = formatted_args
 
     requires_types = {}
     if nested:
@@ -281,8 +291,10 @@ def nested_args_provider(
 def _escape(s):
     return s.replace("%", "%%")
 
-def _format_target(target, fail = fail):
+def _format_target(target, arg, allow_variables, fail = fail):
     if VariableInfo in target:
+        if not allow_variables:
+            fail("Unsupported cc_variable substitution %s in %r." % (target.label, arg))
         return "%%{%s}" % target[VariableInfo].name
     elif DirectoryInfo in target:
         return _escape(target[DirectoryInfo].path)
@@ -293,10 +305,49 @@ def _format_target(target, fail = fail):
 
     fail("%s should be either a variable, a directory, or a single file." % target.label)
 
-def format_args(args, format, must_use = [], fail = fail):
+def _format_string(arg, format, used_vars, allow_variables, fail = fail):
+    upto = 0
+    out = []
+    has_format = False
+
+    # This should be "while true".
+    # This number is used because it's an upper bound of the number of iterations.
+    for _ in range(len(arg)):
+        if upto >= len(arg):
+            break
+
+        # Escaping via "{{" and "}}"
+        if arg[upto] in "{}" and upto + 1 < len(arg) and arg[upto + 1] == arg[upto]:
+            out.append(arg[upto])
+            upto += 2
+        elif arg[upto] == "{":
+            chunks = arg[upto + 1:].split("}", 1)
+            if len(chunks) != 2:
+                fail("Unmatched { in %r" % arg)
+            variable = chunks[0]
+
+            if variable not in format:
+                fail('Unknown variable %r in format string %r. Try using cc_args(..., format = {"//path/to:variable": %r})' % (variable, arg, variable))
+            elif has_format:
+                fail("The format string %r contained multiple variables, which is unsupported." % arg)
+            else:
+                used_vars[variable] = None
+                has_format = True
+                out.append(_format_target(format[variable], arg, allow_variables, fail = fail))
+                upto += len(variable) + 2
+
+        elif arg[upto] == "}":
+            fail("Unexpected } in %r" % arg)
+        else:
+            out.append(_escape(arg[upto]))
+            upto += 1
+
+    return "".join(out)
+
+def format_list(args, format, must_use = [], fail = fail):
     """Lists all of the variables referenced by an argument.
 
-    Eg: format_args(["--foo", "--bar={bar}"], {"bar": VariableInfo(name="bar")})
+    Eg: format_list(["--foo", "--bar={bar}"], {"bar": VariableInfo(name="bar")})
       => ["--foo", "--bar=%{bar}"]
 
     Args:
@@ -312,46 +363,37 @@ def format_args(args, format, must_use = [], fail = fail):
     used_vars = {}
 
     for arg in args:
-        upto = 0
-        out = []
-        has_format = False
-
-        # This should be "while true". I used this number because it's an upper
-        # bound of the number of iterations.
-        for _ in range(len(arg)):
-            if upto >= len(arg):
-                break
-
-            # Escaping via "{{" and "}}"
-            if arg[upto] in "{}" and upto + 1 < len(arg) and arg[upto + 1] == arg[upto]:
-                out.append(arg[upto])
-                upto += 2
-            elif arg[upto] == "{":
-                chunks = arg[upto + 1:].split("}", 1)
-                if len(chunks) != 2:
-                    fail("Unmatched { in %r" % arg)
-                variable = chunks[0]
-
-                if variable not in format:
-                    fail('Unknown variable %r in format string %r. Try using cc_args(..., format = {"//path/to:variable": %r})' % (variable, arg, variable))
-                elif has_format:
-                    fail("The format string %r contained multiple variables, which is unsupported." % arg)
-                else:
-                    used_vars[variable] = None
-                    has_format = True
-                    out.append(_format_target(format[variable], fail = fail))
-                    upto += len(variable) + 2
-
-            elif arg[upto] == "}":
-                fail("Unexpected } in %r" % arg)
-            else:
-                out.append(_escape(arg[upto]))
-                upto += 1
+        formatted.append(_format_string(arg, format, used_vars, True, fail))
+
+    unused_vars = [var for var in must_use if var not in used_vars]
+    if unused_vars:
+        fail("The variable %r was not used in the format string." % unused_vars[0])
+
+    return formatted, used_vars.keys()
+
+def format_dict_values(env, format, must_use = [], fail = fail):
+    """Formats the environment variables.
+
+    Eg: format_dict_values({"FOO": "some/path", "BAR": "{bar}"}, {"bar": DirectoryInfo(path="path/to/bar")})
+      => {"FOO": "some/path", "BAR": "path/to/bar"}
+
+    Args:
+      env: (Dict[str, str]) The environment variables.
+      format: (Dict[str, Target]) A mapping of substitutions from key to target.
+      must_use: (List[str]) A list of substitutions that must be used.
+      fail: The fail function. Used for tests
+
+    Returns:
+      The environment variables with values defined to be compatible with flag groups.
+    """
+    formatted = {}
+    used_vars = {}
 
-        formatted.append("".join(out))
+    for key, value in env.items():
+        formatted[key] = _format_string(value, format, used_vars, False, fail)
 
     unused_vars = [var for var in must_use if var not in used_vars]
     if unused_vars:
         fail("The variable %r was not used in the format string." % unused_vars[0])
 
-    return formatted
+    return formatted, used_vars.keys()

docs/toolchain_api.md

@@ -662,7 +662,7 @@ For more extensive examples, see the usages here:
 | <a id="cc_args-args"></a>args |  (List[str]) The command-line arguments that are applied by using this rule. This is mutually exclusive with [nested](#cc_args-nested).   |  `None` |
 | <a id="cc_args-data"></a>data |  (List[Label]) A list of runtime data dependencies that are required for these arguments to work as intended.   |  `None` |
 | <a id="cc_args-env"></a>env |  (Dict[str, str]) Environment variables that should be set when the tool is invoked.   |  `None` |
-| <a id="cc_args-format"></a>format |  (Dict[str, Label]) A mapping of format strings to the label of the corresponding [`cc_variable`](#cc_variable) that the value should be pulled from. All instances of `{variable_name}` will be replaced with the expanded value of `variable_name` in this dictionary. The complete list of possible variables can be found in https://github.com/bazelbuild/rules_cc/tree/main/cc/toolchains/variables/BUILD. It is not possible to declare custom variables--these are inherent to Bazel itself.   |  `{}` |
+| <a id="cc_args-format"></a>format |  (Dict[str, Label]) A mapping of format strings to the label of a corresponding target. This target can be a `directory`, `subdirectory`, [`cc_variable`](#cc_variable), or a single file that the value should be pulled from. All instances of `{variable_name}` in the `args` list will be replaced with the expanded value in this dictionary. The complete list of possible variables can be found in https://github.com/bazelbuild/rules_cc/tree/main/cc/toolchains/variables/BUILD. It is not possible to declare custom variables--these are inherent to Bazel itself.   |  `{}` |
 | <a id="cc_args-iterate_over"></a>iterate_over |  (Label) The label of a [`cc_variable`](#cc_variable) that should be iterated over. This is intended for use with built-in variables that are lists.   |  `None` |
 | <a id="cc_args-nested"></a>nested |  (List[Label]) A list of [`cc_nested_args`](#cc_nested_args) rules that should be expanded to command-line arguments when this rule is used. This is mutually exclusive with [args](#cc_args-args).   |  `None` |
 | <a id="cc_args-requires_not_none"></a>requires_not_none |  (Label) The label of a [`cc_variable`](#cc_variable) that should be checked for existence before expanding this rule. If the variable is None, this rule will be ignored.   |  `None` |

tests/rule_based_toolchain/args/BUILD

@@ -1,8 +1,14 @@
 load("@rules_testing//lib:util.bzl", "util")
 load("//cc/toolchains:args.bzl", "cc_args")
+load("//cc/toolchains/impl:variables.bzl", "cc_variable", "types")
 load("//tests/rule_based_toolchain:analysis_test_suite.bzl", "analysis_test_suite")
 load(":args_test.bzl", "TARGETS", "TESTS")
 
+cc_variable(
+    name = "some_variable",
+    type = types.string,
+)
+
 util.helper_target(
     cc_args,
     name = "simple",