Add support for parsing constants from NVIDIA driver in driver_ast_parser.

The `driver_ast_parser` tool is now able to parse constants alongside with
structs. This is done by adding a new `constants` field in the JSON input, and
a `constants` field in the JSON output. This will be used later to detect
changes in constants between driver versions.

Also, the parser's `structs` argument was renamed to `input` to account for
both structs and constants.

PiperOrigin-RevId: 792677998

Author: ayushr2

g3doc/proposals/nvidia_driver_differ.md

@@ -317,14 +317,15 @@ make things simple, the inputs and outputs can be encoded in JSON. Overall,
 interfacing with the parser would go something like this:
 
 ```bash
-./driver_ast_parser --structs=structs.json source_file.cc
+./driver_ast_parser --input=input.json source_file.cc
 ```
 
 Input:
 
 ```json
 {
-    "structs": ["STRUCT", "NAMES", "HERE", ...]
+    "structs": ["STRUCT", "NAMES", "HERE", ...],
+    "constants": ["CONSTANT", "NAMES", "HERE", ...]
 }
 ```
 
@@ -346,6 +347,9 @@ Output:
     // All the typedefs found
     "aliases": {
         "NvHandle": "unsigned int"
+    },
+    "constants": {
+      "CONSTANT_NAME": UINT_VALUE
     }
 }
 ```

pkg/sentry/devices/nvproxy/nvproxy_driver_parity_test.go

@@ -22,7 +22,6 @@ package nvproxy_driver_parity_test
 import (
 	"context"
 	"fmt"
-	"os"
 	"reflect"
 	"regexp"
 	"strconv"
@@ -38,25 +37,21 @@ import (
 	"gvisor.dev/gvisor/tools/nvidia_driver_differ/parser"
 )
 
-func createParserRunner(t *testing.T) (*os.File, *parser.Runner) {
+func createParserRunner(t *testing.T) *parser.Runner {
 	t.Helper()
 
 	// Find the parser binary
 	parserPath, err := testutil.FindFile("tools/nvidia_driver_differ/driver_ast_parser")
 	if err != nil {
 		t.Fatalf("Failed to find driver_ast_parser: %v", err)
 	}
-	parserFile, err := os.Open(parserPath)
-	if err != nil {
-		t.Fatalf("Failed to open driver_ast_parser: %v", err)
-	}
 
-	runner, err := parser.NewRunner((*parser.ParserFile)(parserFile))
+	runner, err := parser.NewRunner(parserPath)
 	if err != nil {
 		t.Fatalf("Failed to create parser runner: %v", err)
 	}
 
-	return parserFile, runner
+	return runner
 }
 
 func getDriverDefs(t *testing.T, runner *parser.Runner, version nvconf.DriverVersion) ([]nvproxy.DriverStruct, *parser.OutputJSON) {
@@ -89,8 +84,7 @@ func TestStructDefinitionParity(t *testing.T) {
 	nvproxy.ForEachSupportDriver(func(version nvconf.DriverVersion, _ nvproxy.Checksums) {
 		t.Run(version.String(), func(t *testing.T) {
 			t.Parallel()
-			f, runner := createParserRunner(t)
-			defer f.Close()
+			runner := createParserRunner(t)
 
 			nvproxyDefs, defs := getDriverDefs(t, runner, version)
 

tools/nvidia_driver_differ/BUILD

@@ -19,7 +19,6 @@ cc_binary(
         "@com_google_absl//absl/strings",
         "@llvm-project//clang:ast",
         "@llvm-project//clang:ast_matchers",
-        "@llvm-project//clang:basic",
         "@llvm-project//clang:tooling",
         "@llvm-project//llvm:Support",
         "@nlohmann_json//:json",

tools/nvidia_driver_differ/driver_ast_parser.cc

@@ -26,10 +26,10 @@
 #include "nlohmann/json.hpp"
 #include "clang/include/clang/AST/ASTContext.h"
 #include "clang/include/clang/AST/Decl.h"
+#include "clang/include/clang/AST/Expr.h"
 #include "clang/include/clang/AST/Type.h"
 #include "clang/include/clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/include/clang/ASTMatchers/ASTMatchers.h"
-#include "clang/include/clang/Basic/SourceManager.h"
 #include "clang/include/clang/Tooling/CommonOptionsParser.h"
 #include "clang/include/clang/Tooling/Tooling.h"
 #include "llvm/include/llvm/Support/Casting.h"
@@ -43,6 +43,7 @@ using clang::ast_matchers::hasName;
 using clang::ast_matchers::hasType;
 using clang::ast_matchers::recordDecl;
 using clang::ast_matchers::typedefDecl;
+using clang::ast_matchers::varDecl;
 
 using clang::ast_matchers::MatchFinder;
 
@@ -51,6 +52,7 @@ using json = nlohmann::json;
 struct DriverStructReporter : public MatchFinder::MatchCallback {
   json RecordDefinitions;
   json TypeAliases;
+  json Constants;
   absl::flat_hash_set<std::string> ParsedTypes;
 
   // This matches the case where a struct is being defined.
@@ -89,13 +91,34 @@ struct DriverStructReporter : public MatchFinder::MatchCallback {
         .bind("typedef_decl");
   }
 
+  auto get_constant_matcher(std::string constant_name) {
+    return varDecl(hasName(constant_name)).bind("constant_decl");
+  }
+
   void run(const MatchFinder::MatchResult &result) override {
     const auto *ctx = result.Context;
 
+    const auto *constant_decl =
+        result.Nodes.getNodeAs<clang::VarDecl>("constant_decl");
+    if (constant_decl) {
+      std::string name = constant_decl->getNameAsString();
+      if (!constant_decl->hasInit()) {
+        std::cerr << "Constant " << name << " has no initializer\n";
+        exit(1);
+      }
+      clang::Expr::EvalResult value;
+      if (!constant_decl->getInit()->EvaluateAsInt(value, *ctx)) {
+        std::cerr << "Unable to evaluate constant " << name << "\n";
+        exit(1);
+      }
+      Constants[name] = value.Val.getInt().getZExtValue();
+      return;
+    }
+
     const auto *typedef_decl =
         result.Nodes.getNodeAs<clang::TypedefDecl>("typedef_decl");
     if (typedef_decl == nullptr) {
-      std::cerr << "Unable to find typedef decl\n";
+      std::cerr << "Unable to find a matched declaration\n";
       exit(1);
     }
     std::string name = typedef_decl->getNameAsString();
@@ -218,10 +241,10 @@ static llvm::cl::extrahelp CommonHelp(
     clang::tooling::CommonOptionsParser::HelpMessage);
 static llvm::cl::extrahelp MoreHelp(ToolHelpDescription);
 
-static llvm::cl::opt<std::string> StructNames(
-    "structs",
-    llvm::cl::desc(
-        "Path to the input file containing the struct names to parse."),
+static llvm::cl::opt<std::string> InputFile(
+    "input",
+    llvm::cl::desc("Path to the input file containing the struct and constant "
+                   "names to parse."),
     llvm::cl::cat(DriverASTParserCategory), llvm::cl::Required);
 
 static llvm::cl::opt<std::string> OutputFile(
@@ -247,25 +270,33 @@ int main(int argc, const char **argv) {
   MatchFinder finder;
 
   // Read from StructNames file.
-  std::ifstream StructNamesIS(StructNames);
-  if (!StructNamesIS) {
-    std::cerr << "Unable to open struct names file: " << StructNames << "\n";
+  std::ifstream InputFileIS(InputFile);
+  if (!InputFileIS) {
+    std::cerr << "Unable to open struct names file: " << InputFile << "\n";
     return 1;
   }
-  json StructNamesJSON;
-  StructNamesIS >> StructNamesJSON;
-  for (json::iterator it = StructNamesJSON["structs"].begin();
-       it != StructNamesJSON["structs"].end(); ++it) {
+  json input;
+  InputFileIS >> input;
+  for (json::iterator it = input["structs"].begin();
+       it != input["structs"].end(); ++it) {
     finder.addMatcher(reporter.get_struct_definition_matcher(*it), &reporter);
     finder.addMatcher(reporter.get_struct_typedef_matcher(*it), &reporter);
   }
 
+  if (input.contains("constants")) {
+    for (json::iterator it = input["constants"].begin();
+         it != input["constants"].end(); ++it) {
+      finder.addMatcher(reporter.get_constant_matcher(*it), &reporter);
+    }
+  }
+
   // Run tool
   int ret = Tool.run(clang::tooling::newFrontendActionFactory(&finder).get());
 
   // Print output.
   json output = json::object({{"records", reporter.RecordDefinitions},
-                              {"aliases", reporter.TypeAliases}});
+                              {"aliases", reporter.TypeAliases},
+                              {"constants", reporter.Constants}});
   if (OutputFile.empty()) {
     std::cout << output.dump() << "\n";
   } else {

tools/nvidia_driver_differ/driver_ast_parser.h

@@ -17,9 +17,9 @@
 
 const char ToolHelpDescription[] =
     R"a(This tool parses a given C++ source file and outputs relevant definitions
-for a list of provided struct names. It finds the definitions for each struct
-given by the names provided, as well as any nested structs or types that they
-depend on.
+for a list of provided struct names and constant names. It finds the definitions
+for each struct given by the names provided, as well as any nested structs or
+types that they depend on.
 
 This tool is intended to be used to parse the NVIDIA driver source code; as
 such, there are some assumptions made about how structs are defined and what
@@ -30,14 +30,15 @@ that includes all the files to be parsed. You will also need a
 compile_commands.json file that contains a compile command with the relevant
 include directories.
 
-The struct names should be specified in a JSON file containing a JSON object,
-which has a "structs" key that maps to a list of strings. The tool will search
-for the struct definition in the given source files, and output the struct
-definition to the specified output file.
+The struct and constant names should be specified in a JSON file containing a
+JSON object, which has "structs" and "constants" keys that map to a list of
+strings. The tool will search for their definition in the given source files,
+and output their definition to the specified output file.
 
 This output file will contain a JSON object with a "records" (structs or unions)
-field mapping each name to its definition, as well as an "aliases" field for
-any aliases that were found. A variety of information is outputted:
+field mapping each name to its definition, a "aliases" field for any aliases
+that were found, as well as a "constants" field mapping each name to its value.
+A variety of information is outputted:
 - For records, the fields are given as a JSON array of objects with "name",
   "type", and "offset" keys. The record also has a "size" key indicating the
   size of the struct in bytes, an "is_union" key indicating whether it is a
@@ -46,13 +47,17 @@ any aliases that were found. A variety of information is outputted:
 - For aliases, the type is given as a JSON object with a "type" and "size" key
 
 Example usage:
-    driver_ast_parser --structs=structs.json -o=output.json driver_source_files.h
+    driver_ast_parser --input=input.json -o=output.json driver_source_files.h
 
-structs.json:
+input.json:
     {
         "structs": [
             "TestStruct",
             "TestStruct2"
+        ],
+        "constants": [
+            "TEST_CONSTANT",
+            "ANOTHER_CONSTANT"
         ]
     }
 )a";

tools/nvidia_driver_differ/driver_ast_parser_test.go

@@ -42,34 +42,44 @@ func TestParser(t *testing.T) {
 	}
 
 	// Write a file containing the struct name we want to parse.
-	structsFile, err := os.CreateTemp(os.TempDir(), "structs.*.json")
+	inputFile, err := os.CreateTemp(os.TempDir(), "input.*.json")
 	if err != nil {
-		t.Fatalf("failed to create structs file: %v", err)
+		t.Fatalf("failed to create input file: %v", err)
 	}
 	defer func() {
-		if err := structsFile.Close(); err != nil {
-			t.Fatalf("failed to close structs file: %v", err)
+		if err := inputFile.Close(); err != nil {
+			t.Fatalf("failed to close input file: %v", err)
 		}
-		if err := os.Remove(structsFile.Name()); err != nil {
-			t.Fatalf("failed to remove structs file: %v", err)
+		if err := os.Remove(inputFile.Name()); err != nil {
+			t.Fatalf("failed to remove input file: %v", err)
 		}
 	}()
 
 	input := parser.InputJSON{
 		Structs: []string{"TestStruct", "TestStruct2"},
+		Constants: []string{
+			"VAR_CONSTANT_MACRO",
+			"VAR_ADDITION_MACRO",
+			"VAR_UNSIGNED_HEX_MACRO",
+			"VAR_PARENTHESIZED_HEX_MACRO",
+			"VAR_USES_FUNCTION_MACRO",
+		},
 	}
-	if err := json.NewEncoder(structsFile).Encode(&input); err != nil {
-		t.Fatalf("failed to write input structs file: %v", err)
+	if err := json.NewEncoder(inputFile).Encode(&input); err != nil {
+		t.Fatalf("failed to write input input file: %v", err)
 	}
-	structsFile.Sync()
+	inputFile.Sync()
 
-	cmd := exec.Command(driverParser, "--structs", structsFile.Name(), testStructFile)
+	cmd := exec.Command(driverParser, "--input", inputFile.Name(), testStructFile)
 	var stderr strings.Builder
 	cmd.Stderr = &stderr
 	out, err := cmd.Output()
 	if err != nil {
 		t.Fatalf("failed to run driver_ast_parser: %v\n%s", err, stderr.String())
 	}
+	if stderr.Len() > 0 {
+		t.Logf("driver_ast_parser stderr:\n%s", stderr.String())
+	}
 
 	outputJSON := parser.OutputJSON{}
 	if err := json.Unmarshal(out, &outputJSON); err != nil {
@@ -122,6 +132,13 @@ func TestParser(t *testing.T) {
 			"OtherInt": parser.TypeDef{Type: "int", Size: 4},
 			"int":      parser.TypeDef{Type: "int", Size: 4},
 		},
+		Constants: map[string]uint64{
+			"VAR_CONSTANT_MACRO":          0x1469,
+			"VAR_ADDITION_MACRO":          0x1470,
+			"VAR_UNSIGNED_HEX_MACRO":      0x279,
+			"VAR_PARENTHESIZED_HEX_MACRO": 0x50a0,
+			"VAR_USES_FUNCTION_MACRO":     0x1,
+		},
 	}
 
 	if diff := cmp.Diff(expectedOutput, outputJSON, cmpopts.IgnoreFields(parser.RecordDef{}, "Source")); diff != "" {

tools/nvidia_driver_differ/parser/json_definitions.go

@@ -22,18 +22,19 @@ import (
 	"strings"
 
 	"github.com/google/go-cmp/cmp"
-	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
 )
 
-// InputJSON is the format for the structs.json file that driver_ast_parser takes as input.
+// InputJSON is the format for the input.json file that driver_ast_parser takes as input.
 type InputJSON struct {
-	Structs []string `json:"structs"`
+	Structs   []string `json:"structs"`
+	Constants []string `json:"constants"`
 }
 
 // OutputJSON is the format for the output of driver_ast_parser.
 type OutputJSON struct {
-	Records RecordDefs
-	Aliases TypeAliases
+	Records   RecordDefs
+	Aliases   TypeAliases
+	Constants map[string]uint64
 }
 
 // Merge merges the struct definitions from b into this OutputJSON.
@@ -44,8 +45,12 @@ func (a *OutputJSON) Merge(b OutputJSON) {
 	if a.Aliases == nil {
 		a.Aliases = make(TypeAliases)
 	}
+	if a.Constants == nil {
+		a.Constants = make(map[string]uint64)
+	}
 	maps.Copy(a.Records, b.Records)
 	maps.Copy(a.Aliases, b.Aliases)
+	maps.Copy(a.Constants, b.Constants)
 }
 
 // RecordField represents a field in a record (struct or union).
@@ -85,7 +90,7 @@ type RecordDefs map[string]RecordDef
 type TypeAliases map[string]TypeDef
 
 // GetRecordDiff prints a diff between two records.
-func GetRecordDiff(name nvproxy.DriverStructName, a, b RecordDef) string {
+func GetRecordDiff(name string, a, b RecordDef) string {
 	var sb strings.Builder
 	fmt.Fprintf(&sb, "--- A: %s\n", a.Source)
 	fmt.Fprintf(&sb, "+++ B: %s\n", b.Source)