merge main into amd-staging (#629)

Author: z1-cciauto

.github/workflows/commit-access-review.py

@@ -170,80 +170,6 @@ def get_num_commits(gh: github.Github, user: str, start_date: datetime.datetime)
     return count
 
 
-def is_new_committer_query_repo(
-    gh: github.Github, user: str, start_date: datetime.datetime
-) -> bool:
-    """
-    Determine if ``user`` is a new committer.  A new committer can keep their
-    commit access even if they don't meet the criteria.
-    """
-    variables = {
-        "user": user,
-    }
-
-    user_query = """
-        query ($user: String!) {
-          user(login: $user) {
-            id
-          }
-        }
-    """
-
-    res_header, res_data = gh._Github__requester.graphql_query(
-        query=user_query, variables=variables
-    )
-    data = res_data["data"]
-    variables["owner"] = "llvm"
-    variables["user_id"] = data["user"]["id"]
-    variables["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%S")
-
-    query = """
-        query ($owner: String!, $user_id: ID!){
-          organization(login: $owner) {
-            repository(name: "llvm-project") {
-              ref(qualifiedName: "main") {
-                target {
-                  ... on Commit {
-                    history(author: {id: $user_id }, first: 5) {
-                      nodes {
-                        committedDate
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-     """
-
-    res_header, res_data = gh._Github__requester.graphql_query(
-        query=query, variables=variables
-    )
-    data = res_data["data"]
-    repo = data["organization"]["repository"]
-    commits = repo["ref"]["target"]["history"]["nodes"]
-    if len(commits) == 0:
-        return True
-    committed_date = commits[-1]["committedDate"]
-    if datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ") < start_date:
-        return False
-    return True
-
-
-def is_new_committer(
-    gh: github.Github, user: str, start_date: datetime.datetime
-) -> bool:
-    """
-    Wrapper around is_new_commiter_query_repo to handle exceptions.
-    """
-    try:
-        return is_new_committer_query_repo(gh, user, start_date)
-    except:
-        pass
-    return True
-
-
 def get_review_count(
     gh: github.Github, user: str, start_date: datetime.datetime
 ) -> int:
@@ -383,13 +309,6 @@ def main():
 
     print("After Commits:", len(triage_list), "triagers")
 
-    # Step 4 check for new committers
-    for user in list(triage_list.keys()):
-        print("Checking", user)
-        if is_new_committer(gh, user, one_year_ago):
-            print("Removing new committer: ", user)
-            del triage_list[user]
-
     print("Complete:", len(triage_list), "triagers")
 
     with open("triagers.log", "w") as triagers_log:

.github/workflows/hlsl-test-all.yaml

@@ -52,11 +52,6 @@ jobs:
           repository: llvm/offload-golden-images
           ref: main
           path: golden-images
-      - name: Setup Windows
-        if: runner.os == 'Windows'
-        uses: llvm/actions/setup-windows@5dd955034a6742a2e21d82bf165fcb1050ae7b49 # main
-        with:
-          arch: amd64
       - name: Build DXC
         run: |
             cd DXC

.github/workflows/libclang-abi-tests.yml

@@ -101,8 +101,6 @@ jobs:
             ref: ${{ github.sha }}
             repo: ${{ github.repository }}
     steps:
-      - name: Install Ninja
-        uses: llvm/actions/install-ninja@5dd955034a6742a2e21d82bf165fcb1050ae7b49 # main
       - name: Download source code
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:

clang-tools-extra/clang-doc/Generators.cpp

@@ -7,9 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "Generators.h"
+#include "support/File.h"
+#include "llvm/Support/TimeProfiler.h"
 
 LLVM_INSTANTIATE_REGISTRY(clang::doc::GeneratorRegistry)
 
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
 namespace clang {
 namespace doc {
 
@@ -42,6 +48,136 @@ std::string getTagType(TagTypeKind AS) {
   llvm_unreachable("Unknown TagTypeKind");
 }
 
+Error createFileOpenError(StringRef FileName, std::error_code EC) {
+  return createFileError("cannot open file " + FileName, EC);
+}
+
+Error MustacheGenerator::setupTemplate(
+    std::unique_ptr<MustacheTemplateFile> &Template, StringRef TemplatePath,
+    std::vector<std::pair<StringRef, StringRef>> Partials) {
+  auto T = MustacheTemplateFile::createMustacheFile(TemplatePath);
+  if (Error Err = T.takeError())
+    return Err;
+  Template = std::move(T.get());
+  for (const auto &[Name, FileName] : Partials)
+    if (auto Err = Template->registerPartialFile(Name, FileName))
+      return Err;
+  return Error::success();
+}
+
+Error MustacheGenerator::generateDocumentation(
+    StringRef RootDir, StringMap<std::unique_ptr<doc::Info>> Infos,
+    const clang::doc::ClangDocContext &CDCtx, std::string DirName) {
+  {
+    llvm::TimeTraceScope TS("Setup Templates");
+    if (auto Err = setupTemplateFiles(CDCtx))
+      return Err;
+  }
+
+  {
+    llvm::TimeTraceScope TS("Generate JSON for Mustache");
+    if (auto JSONGenerator = findGeneratorByName("json")) {
+      if (Error Err = JSONGenerator.get()->generateDocumentation(
+              RootDir, std::move(Infos), CDCtx))
+        return Err;
+    } else
+      return JSONGenerator.takeError();
+  }
+
+  SmallString<128> JSONPath;
+  sys::path::native(RootDir.str() + "/json", JSONPath);
+
+  {
+    llvm::TimeTraceScope TS("Iterate JSON files");
+    std::error_code EC;
+    sys::fs::recursive_directory_iterator JSONIter(JSONPath, EC);
+    std::vector<json::Value> JSONFiles;
+    JSONFiles.reserve(Infos.size());
+    if (EC)
+      return createStringError("Failed to create directory iterator.");
+
+    SmallString<128> DocsDirPath(RootDir.str() + '/' + DirName);
+    sys::path::native(DocsDirPath);
+    if (auto EC = sys::fs::create_directories(DocsDirPath))
+      return createFileError(DocsDirPath, EC);
+    while (JSONIter != sys::fs::recursive_directory_iterator()) {
+      // create the same directory structure in the docs format dir
+      if (JSONIter->type() == sys::fs::file_type::directory_file) {
+        SmallString<128> DocsClonedPath(JSONIter->path());
+        sys::path::replace_path_prefix(DocsClonedPath, JSONPath, DocsDirPath);
+        if (auto EC = sys::fs::create_directories(DocsClonedPath)) {
+          return createFileError(DocsClonedPath, EC);
+        }
+      }
+
+      if (EC)
+        return createFileError("Failed to iterate: " + JSONIter->path(), EC);
+
+      auto Path = StringRef(JSONIter->path());
+      if (!Path.ends_with(".json")) {
+        JSONIter.increment(EC);
+        continue;
+      }
+
+      auto File = MemoryBuffer::getFile(Path);
+      if (EC = File.getError(); EC) {
+        // TODO: Buffer errors to report later, look into using Clang
+        // diagnostics.
+        llvm::errs() << "Failed to open file: " << Path << " " << EC.message()
+                     << '\n';
+      }
+
+      auto Parsed = json::parse((*File)->getBuffer());
+      if (!Parsed)
+        return Parsed.takeError();
+      auto ValidJSON = Parsed.get();
+
+      std::error_code FileErr;
+      SmallString<128> DocsFilePath(JSONIter->path());
+      sys::path::replace_path_prefix(DocsFilePath, JSONPath, DocsDirPath);
+      sys::path::replace_extension(DocsFilePath, DirName);
+      raw_fd_ostream InfoOS(DocsFilePath, FileErr, sys::fs::OF_None);
+      if (FileErr)
+        return createFileOpenError(Path, FileErr);
+
+      auto RelativeRootPath = getRelativePathToRoot(DocsFilePath, DocsDirPath);
+      auto InfoTypeStr =
+          getInfoTypeStr(Parsed->getAsObject(), sys::path::stem(DocsFilePath));
+      if (!InfoTypeStr)
+        return InfoTypeStr.takeError();
+      if (Error Err = generateDocForJSON(*Parsed, InfoOS, CDCtx,
+                                         InfoTypeStr.get(), RelativeRootPath))
+        return Err;
+      JSONIter.increment(EC);
+    }
+  }
+
+  return Error::success();
+}
+
+Expected<std::string> MustacheGenerator::getInfoTypeStr(Object *Info,
+                                                        StringRef Filename) {
+  auto StrValue = (*Info)["InfoType"];
+  if (StrValue.kind() != json::Value::Kind::String)
+    return createStringError("JSON file '%s' does not contain key: 'InfoType'.",
+                             Filename.str().c_str());
+  auto ObjTypeStr = StrValue.getAsString();
+  if (!ObjTypeStr.has_value())
+    return createStringError(
+        "JSON file '%s' does not contain 'InfoType' field as a string.",
+        Filename.str().c_str());
+  return ObjTypeStr.value().str();
+}
+
+SmallString<128>
+MustacheGenerator::getRelativePathToRoot(StringRef PathToFile,
+                                         StringRef DocsRootPath) {
+  SmallString<128> PathVec(PathToFile);
+  // Remove filename, or else the relative path will have an extra "../"
+  sys::path::remove_filename(PathVec);
+  return computeRelativePath(DocsRootPath, PathVec);
+}
+
 llvm::Error Generator::createResources(ClangDocContext &CDCtx) {
   return llvm::Error::success();
 }

clang-tools-extra/clang-doc/Generators.h

@@ -14,6 +14,8 @@
 
 #include "Representation.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Mustache.h"
 #include "llvm/Support/Registry.h"
 
 namespace clang {
@@ -27,10 +29,9 @@ class Generator {
 
   // Write out the decl info for the objects in the given map in the specified
   // format.
-  virtual llvm::Error
-  generateDocs(StringRef RootDir,
-               llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
-               const ClangDocContext &CDCtx) = 0;
+  virtual llvm::Error generateDocumentation(
+      StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+      const ClangDocContext &CDCtx, std::string DirName = "") = 0;
 
   // This function writes a file with the index previously constructed.
   // It can be overwritten by any of the inherited generators.
@@ -52,6 +53,85 @@ findGeneratorByName(llvm::StringRef Format);
 
 std::string getTagType(TagTypeKind AS);
 
+llvm::Error createFileOpenError(StringRef FileName, std::error_code EC);
+
+class MustacheTemplateFile {
+  llvm::BumpPtrAllocator Allocator;
+  llvm::StringSaver Saver;
+  llvm::mustache::MustacheContext Ctx;
+  llvm::mustache::Template T;
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+
+public:
+  static Expected<std::unique_ptr<MustacheTemplateFile>>
+  createMustacheFile(StringRef FileName) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrError =
+        llvm::MemoryBuffer::getFile(FileName);
+    if (auto EC = BufferOrError.getError())
+      return createFileOpenError(FileName, EC);
+    return std::make_unique<MustacheTemplateFile>(
+        std::move(BufferOrError.get()));
+  }
+
+  llvm::Error registerPartialFile(StringRef Name, StringRef FileName) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrError =
+        llvm::MemoryBuffer::getFile(FileName);
+    if (auto EC = BufferOrError.getError())
+      return createFileOpenError(FileName, EC);
+
+    std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrError.get());
+    StringRef FileContent = Buffer->getBuffer();
+    T.registerPartial(Name.str(), FileContent.str());
+    return llvm::Error::success();
+  }
+
+  void render(llvm::json::Value &V, raw_ostream &OS) { T.render(V, OS); }
+
+  MustacheTemplateFile(std::unique_ptr<llvm::MemoryBuffer> &&B)
+      : Saver(Allocator), Ctx(Allocator, Saver), T(B->getBuffer(), Ctx),
+        Buffer(std::move(B)) {}
+};
+
+struct MustacheGenerator : public Generator {
+  Expected<std::string> getInfoTypeStr(llvm::json::Object *Info,
+                                       StringRef Filename);
+
+  /// Used to find the relative path from the file to the format's docs root.
+  /// Mainly used for the HTML resource paths.
+  SmallString<128> getRelativePathToRoot(StringRef PathToFile,
+                                         StringRef DocsRootPath);
+  virtual ~MustacheGenerator() = default;
+
+  /// Initializes the template files from disk and calls setupTemplate to
+  /// register partials
+  virtual llvm::Error setupTemplateFiles(const ClangDocContext &CDCtx) = 0;
+
+  /// Populates templates with data from JSON and calls any specifics for the
+  /// format. For example, for HTML it will render the paths for CSS and JS.
+  virtual llvm::Error generateDocForJSON(llvm::json::Value &JSON,
+                                         llvm::raw_fd_ostream &OS,
+                                         const ClangDocContext &CDCtx,
+                                         StringRef ObjectTypeStr,
+                                         StringRef RelativeRootPath) = 0;
+
+  /// Registers partials to templates.
+  llvm::Error
+  setupTemplate(std::unique_ptr<MustacheTemplateFile> &Template,
+                StringRef TemplatePath,
+                std::vector<std::pair<StringRef, StringRef>> Partials);
+
+  /// \brief The main orchestrator for Mustache-based documentation.
+  ///
+  /// 1. Initializes templates files from disk by calling setupTemplateFiles.
+  /// 2. Calls the JSON generator to write JSON to disk.
+  /// 3. Iterates over the JSON files, recreates the directory structure from
+  /// JSON, and calls generateDocForJSON for each file.
+  /// 4. A file of the desired format is created.
+  llvm::Error generateDocumentation(
+      StringRef RootDir, llvm::StringMap<std::unique_ptr<doc::Info>> Infos,
+      const clang::doc::ClangDocContext &CDCtx, std::string DirName) override;
+};
+
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the generators.
 extern volatile int YAMLGeneratorAnchorSource;