Merge branch 'main' into mlir-scf2gpu-affineapply-crash

Author: shashforge

.github/workflows/build-metrics-container.yml

@@ -21,39 +21,19 @@ jobs:
   build-metrics-container:
     if: github.repository_owner == 'llvm'
     runs-on: ubuntu-24.04
-    outputs:
-      container-name: ${{ steps.vars.outputs.container-name }}
-      container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
-      container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
-          sparse-checkout: .ci/metrics/
-      - name: Write Variables
-        id: vars
-        run: |
-          tag=`date +%s`
-          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/metrics"
-          echo "container-name=$container_name" >> $GITHUB_OUTPUT
-          echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
-          echo "container-filename=$(echo $container_name:$tag  | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
+          sparse-checkout: |
+            .ci/metrics/
+            .github/actions/build-container
       - name: Build Container
-        working-directory: ./.ci/metrics
-        run: |
-          podman build -t ${{ steps.vars.outputs.container-name-tag }} -f Dockerfile .
-      # Save the container so we have it in case the push fails.  This also
-      # allows us to separate the push step into a different job so we can
-      # maintain minimal permissions while building the container.
-      - name: Save Container Image
-        run: |
-          podman save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
-      - name: Upload Container Image
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        uses: ./.github/actions/build-container
         with:
-          name: container
-          path: ${{ steps.vars.outputs.container-filename }}
-          retention-days: 14
+          container-name: metrics
+          context: .ci/metrics
+          dockerfile: .ci/metrics/Dockerfile
 
   push-metrics-container:
     if: github.event_name == 'push'
@@ -65,14 +45,12 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Download Container
-        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+      - name: Checkout LLVM
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          sparse-checkout: |
+            .github/actions/push-container
+
+      - uses: ./.github/actions/push-container
         with:
-          name: container
-      - name: Push Container
-        run: |
-          podman load -i ${{ needs.build-metrics-container.outputs.container-filename }}
-          podman tag ${{ needs.build-metrics-container.outputs.container-name-tag }} ${{ needs.build-metrics-container.outputs.container-name }}:latest
-          podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
-          podman push ${{ needs.build-metrics-container.outputs.container-name-tag }}
-          podman push ${{ needs.build-metrics-container.outputs.container-name }}:latest
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/commit-access-review.py

@@ -170,80 +170,6 @@ def get_num_commits(gh: github.Github, user: str, start_date: datetime.datetime)
     return count
 
 
-def is_new_committer_query_repo(
-    gh: github.Github, user: str, start_date: datetime.datetime
-) -> bool:
-    """
-    Determine if ``user`` is a new committer.  A new committer can keep their
-    commit access even if they don't meet the criteria.
-    """
-    variables = {
-        "user": user,
-    }
-
-    user_query = """
-        query ($user: String!) {
-          user(login: $user) {
-            id
-          }
-        }
-    """
-
-    res_header, res_data = gh._Github__requester.graphql_query(
-        query=user_query, variables=variables
-    )
-    data = res_data["data"]
-    variables["owner"] = "llvm"
-    variables["user_id"] = data["user"]["id"]
-    variables["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%S")
-
-    query = """
-        query ($owner: String!, $user_id: ID!){
-          organization(login: $owner) {
-            repository(name: "llvm-project") {
-              ref(qualifiedName: "main") {
-                target {
-                  ... on Commit {
-                    history(author: {id: $user_id }, first: 5) {
-                      nodes {
-                        committedDate
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-     """
-
-    res_header, res_data = gh._Github__requester.graphql_query(
-        query=query, variables=variables
-    )
-    data = res_data["data"]
-    repo = data["organization"]["repository"]
-    commits = repo["ref"]["target"]["history"]["nodes"]
-    if len(commits) == 0:
-        return True
-    committed_date = commits[-1]["committedDate"]
-    if datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ") < start_date:
-        return False
-    return True
-
-
-def is_new_committer(
-    gh: github.Github, user: str, start_date: datetime.datetime
-) -> bool:
-    """
-    Wrapper around is_new_commiter_query_repo to handle exceptions.
-    """
-    try:
-        return is_new_committer_query_repo(gh, user, start_date)
-    except:
-        pass
-    return True
-
-
 def get_review_count(
     gh: github.Github, user: str, start_date: datetime.datetime
 ) -> int:
@@ -383,13 +309,6 @@ def main():
 
     print("After Commits:", len(triage_list), "triagers")
 
-    # Step 4 check for new committers
-    for user in list(triage_list.keys()):
-        print("Checking", user)
-        if is_new_committer(gh, user, one_year_ago):
-            print("Removing new committer: ", user)
-            del triage_list[user]
-
     print("Complete:", len(triage_list), "triagers")
 
     with open("triagers.log", "w") as triagers_log:

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -90,7 +90,7 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-ARG RUNNER_VERSION=2.329.0
+ARG RUNNER_VERSION=2.330.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \

.github/workflows/containers/github-action-ci/Dockerfile

@@ -99,7 +99,7 @@ WORKDIR /home/gha
 
 FROM ci-container AS ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.329.0
+ENV GITHUB_RUNNER_VERSION=2.330.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \

.github/workflows/hlsl-test-all.yaml

@@ -52,11 +52,6 @@ jobs:
           repository: llvm/offload-golden-images
           ref: main
           path: golden-images
-      - name: Setup Windows
-        if: runner.os == 'Windows'
-        uses: llvm/actions/setup-windows@5dd955034a6742a2e21d82bf165fcb1050ae7b49 # main
-        with:
-          arch: amd64
       - name: Build DXC
         run: |
             cd DXC

.github/workflows/libclang-abi-tests.yml

@@ -101,8 +101,6 @@ jobs:
             ref: ${{ github.sha }}
             repo: ${{ github.repository }}
     steps:
-      - name: Install Ninja
-        uses: llvm/actions/install-ninja@5dd955034a6742a2e21d82bf165fcb1050ae7b49 # main
       - name: Download source code
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:

clang-tools-extra/clang-doc/Generators.cpp

@@ -7,9 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "Generators.h"
+#include "support/File.h"
+#include "llvm/Support/TimeProfiler.h"
 
 LLVM_INSTANTIATE_REGISTRY(clang::doc::GeneratorRegistry)
 
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
 namespace clang {
 namespace doc {
 
@@ -42,6 +48,136 @@ std::string getTagType(TagTypeKind AS) {
   llvm_unreachable("Unknown TagTypeKind");
 }
 
+Error createFileOpenError(StringRef FileName, std::error_code EC) {
+  return createFileError("cannot open file " + FileName, EC);
+}
+
+Error MustacheGenerator::setupTemplate(
+    std::unique_ptr<MustacheTemplateFile> &Template, StringRef TemplatePath,
+    std::vector<std::pair<StringRef, StringRef>> Partials) {
+  auto T = MustacheTemplateFile::createMustacheFile(TemplatePath);
+  if (Error Err = T.takeError())
+    return Err;
+  Template = std::move(T.get());
+  for (const auto &[Name, FileName] : Partials)
+    if (auto Err = Template->registerPartialFile(Name, FileName))
+      return Err;
+  return Error::success();
+}
+
+Error MustacheGenerator::generateDocumentation(
+    StringRef RootDir, StringMap<std::unique_ptr<doc::Info>> Infos,
+    const clang::doc::ClangDocContext &CDCtx, std::string DirName) {
+  {
+    llvm::TimeTraceScope TS("Setup Templates");
+    if (auto Err = setupTemplateFiles(CDCtx))
+      return Err;
+  }
+
+  {
+    llvm::TimeTraceScope TS("Generate JSON for Mustache");
+    if (auto JSONGenerator = findGeneratorByName("json")) {
+      if (Error Err = JSONGenerator.get()->generateDocumentation(
+              RootDir, std::move(Infos), CDCtx))
+        return Err;
+    } else
+      return JSONGenerator.takeError();
+  }
+
+  SmallString<128> JSONPath;
+  sys::path::native(RootDir.str() + "/json", JSONPath);
+
+  {
+    llvm::TimeTraceScope TS("Iterate JSON files");
+    std::error_code EC;
+    sys::fs::recursive_directory_iterator JSONIter(JSONPath, EC);
+    std::vector<json::Value> JSONFiles;
+    JSONFiles.reserve(Infos.size());
+    if (EC)
+      return createStringError("Failed to create directory iterator.");
+
+    SmallString<128> DocsDirPath(RootDir.str() + '/' + DirName);
+    sys::path::native(DocsDirPath);
+    if (auto EC = sys::fs::create_directories(DocsDirPath))
+      return createFileError(DocsDirPath, EC);
+    while (JSONIter != sys::fs::recursive_directory_iterator()) {
+      // create the same directory structure in the docs format dir
+      if (JSONIter->type() == sys::fs::file_type::directory_file) {
+        SmallString<128> DocsClonedPath(JSONIter->path());
+        sys::path::replace_path_prefix(DocsClonedPath, JSONPath, DocsDirPath);
+        if (auto EC = sys::fs::create_directories(DocsClonedPath)) {
+          return createFileError(DocsClonedPath, EC);
+        }
+      }
+
+      if (EC)
+        return createFileError("Failed to iterate: " + JSONIter->path(), EC);
+
+      auto Path = StringRef(JSONIter->path());
+      if (!Path.ends_with(".json")) {
+        JSONIter.increment(EC);
+        continue;
+      }
+
+      auto File = MemoryBuffer::getFile(Path);
+      if (EC = File.getError(); EC) {
+        // TODO: Buffer errors to report later, look into using Clang
+        // diagnostics.
+        llvm::errs() << "Failed to open file: " << Path << " " << EC.message()
+                     << '\n';
+      }
+
+      auto Parsed = json::parse((*File)->getBuffer());
+      if (!Parsed)
+        return Parsed.takeError();
+      auto ValidJSON = Parsed.get();
+
+      std::error_code FileErr;
+      SmallString<128> DocsFilePath(JSONIter->path());
+      sys::path::replace_path_prefix(DocsFilePath, JSONPath, DocsDirPath);
+      sys::path::replace_extension(DocsFilePath, DirName);
+      raw_fd_ostream InfoOS(DocsFilePath, FileErr, sys::fs::OF_None);
+      if (FileErr)
+        return createFileOpenError(Path, FileErr);
+
+      auto RelativeRootPath = getRelativePathToRoot(DocsFilePath, DocsDirPath);
+      auto InfoTypeStr =
+          getInfoTypeStr(Parsed->getAsObject(), sys::path::stem(DocsFilePath));
+      if (!InfoTypeStr)
+        return InfoTypeStr.takeError();
+      if (Error Err = generateDocForJSON(*Parsed, InfoOS, CDCtx,
+                                         InfoTypeStr.get(), RelativeRootPath))
+        return Err;
+      JSONIter.increment(EC);
+    }
+  }
+
+  return Error::success();
+}
+
+Expected<std::string> MustacheGenerator::getInfoTypeStr(Object *Info,
+                                                        StringRef Filename) {
+  auto StrValue = (*Info)["InfoType"];
+  if (StrValue.kind() != json::Value::Kind::String)
+    return createStringError("JSON file '%s' does not contain key: 'InfoType'.",
+                             Filename.str().c_str());
+  auto ObjTypeStr = StrValue.getAsString();
+  if (!ObjTypeStr.has_value())
+    return createStringError(
+        "JSON file '%s' does not contain 'InfoType' field as a string.",
+        Filename.str().c_str());
+  return ObjTypeStr.value().str();
+}
+
+SmallString<128>
+MustacheGenerator::getRelativePathToRoot(StringRef PathToFile,
+                                         StringRef DocsRootPath) {
+  SmallString<128> PathVec(PathToFile);
+  // Remove filename, or else the relative path will have an extra "../"
+  sys::path::remove_filename(PathVec);
+  return computeRelativePath(DocsRootPath, PathVec);
+}
+
 llvm::Error Generator::createResources(ClangDocContext &CDCtx) {
   return llvm::Error::success();
 }