Add verification step to create_draft_release to check SSI one-pipeline succeeded (#5865 -> v2) (#5879)

## Summary of changes

Adds a stage to `create_draft_release` that verifies the gitlab stages
all passed before starting the release

## Reason for change

Before we merged #5818 we were testing the lib injection images prior to
publishing, but that PR removed the checks (we're building/testing
everything in the one pipeline instead)

This PR, _explicitly_ checks that everything passed in GitLab before
starting the release.

## Implementation details

Pulls all the statuses for the commit, and makes sure the GitLab ones
all have a passing status.

Pros: 
- If/when the one-pipeline expands, we automatically check for success.
- One pipeline can change (add/remove checks) and we automatically make
sure they all passed before doing a release.

Cons: 
- The one pipeline could break at some point without being a _real_
issue, which would block our releases. So added an override we can use
in case of emergency. It should only be used when the reporting is
"false" though, because the one pipeline creates and publishes the SSI
artifacts.
- We don't explicitly require any of the checks. This gives us
flexibility (they can add or remove checks) but we won't know if we're
actually checking them all (for example)
- If the GitHub/GitLab link isn't working (due to issues either side),
we might not be notified about failures

## Test coverage

Tested locally to confirm the `VerifyReleaseReadiness` works as
expected. e.g. run

```powershell
.\tracer\build.ps1 VerifyReleaseReadiness -CommitSha 89bf7b3 -GITHUB_TOKEN <your token>
```

and it passes, but `ec735117ceaa963d5033f83ed80daaa88f970867` fails (for
example)

## Other details

Backport of #5865

---------

Co-authored-by: Kevin Gosse <[email protected]>

Author: andrewlock

.github/workflows/create_draft_release.yml

@@ -6,6 +6,9 @@ on:
       forced_commit_id:
         description: 'Force using artifacts from specific commit? If provided, this will try and use the artifacts from the given commit, regardless of build status'
         required: false
+      ignore_gitlab_failures:
+        description: "DANGER Force ignoring any issues with the GitLab artifacts or SSI. Don't use this unless you _really_ know what you're doing"
+        required: false
 
 jobs:
   create_draft_release:
@@ -40,6 +43,12 @@ jobs:
           git config user.name "${{ github.actor }}"
           git config user.email "${{ github.actor }}@users.noreply.github.com"
 
+      - name: "Check GitLab status"
+        if: ${{ !github.event.inputs.ignore_gitlab_failures }}
+        run: ./tracer/build.sh VerifyReleaseReadiness
+        env:
+          CommitSha: "${{ steps.set_sha.outputs.sha }}"
+
       - name: "Get current version"
         id: versions
         run: ./tracer/build.sh OutputCurrentVersionToGitHub

.nuke/build.schema.json

@@ -464,6 +464,7 @@
               "UpdateVendoredCode",
               "UpdateVersion",
               "VerifyChangedFilesFromVersionBump",
+              "VerifyReleaseReadiness",
               "ZipMonitoringHome",
               "ZipMonitoringHomeLinux",
               "ZipMonitoringHomeOsx",
@@ -677,6 +678,7 @@
               "UpdateVendoredCode",
               "UpdateVersion",
               "VerifyChangedFilesFromVersionBump",
+              "VerifyReleaseReadiness",
               "ZipMonitoringHome",
               "ZipMonitoringHomeLinux",
               "ZipMonitoringHomeOsx",

tracer/build/_build/Build.GitHub.cs

@@ -17,6 +17,7 @@
 using Nuke.Common;
 using Nuke.Common.IO;
 using Nuke.Common.Tooling;
+using Nuke.Common.Tools.Docker;
 using Nuke.Common.Tools.Git;
 using Octokit;
 using Octokit.GraphQL;
@@ -1103,6 +1104,77 @@ await client.Issue.Milestone.Update(
              }
          });
 
+    Target VerifyReleaseReadiness => _ => _
+            .Unlisted()
+            .Requires(() => GitHubToken)
+            .Requires(() => CommitSha)
+            .Executes(async () =>
+            {
+                Logger.Information("Verifying SSI artifact build succeeded for commit {Commit}...", CommitSha);
+                var client = GetGitHubClient();
+                var statuses = await client.Repository.Status.GetAll(
+                    owner: GitHubRepositoryOwner,
+                    name: GitHubRepositoryName,
+                    reference: CommitSha);
+
+                // find all the gitlab-related SSI statuses, they _all_ need to have passed
+                // (apart from the serverless one, we'll ignore that for now)
+                // This includes the _full_ list, so we just want to check that we have a success for each unique job
+                var ssiStatuses = statuses
+                    .Where(x => x.Context.StartsWith("dd-gitlab/") && x.Context != "dd-gitlab/benchmark-serverless")
+                    .ToLookup(x => x.Context, x => x);
+
+                if (ssiStatuses.Count == 0)
+                {
+                    throw new Exception("No GitLab builds for SSI artifacts found. Please check the commit and try again");
+                }
+
+                var failedSsi = ssiStatuses
+                    .Where(x => !x.Any(status => status.State == CommitState.Success))
+                    .ToList();
+
+                if (failedSsi.Any())
+                {
+                    Logger.Warning("The following gitlab jobs did not complete successfully. Please check the builds for details about why");
+                    foreach (var failed in failedSsi)
+                    {
+                        var build = failed.OrderBy(c => c.State.Value).First();
+                        Logger.Warning("- {Job} ({Status}) {Link}", failed.Key, build.State, build.TargetUrl);
+                    }
+                    
+                    throw new Exception("Some gitlab jobs did not build/test successfully. Please check the builds for details about why.");
+                }
+
+                var stages = string.Join(", ", ssiStatuses.Select(x => x.Key));
+                Logger.Information("All gitlab build stages ({Stages}) completed successfully", stages);
+                
+                // assert that the docker image for the commit is present
+                var image = $"ghcr.io/datadog/dd-trace-dotnet/dd-lib-dotnet-init:{CommitSha}";
+                VerifyDockerImageExists(image);
+                
+                if(new Version(Version).Major < 3)
+                {
+                    image = $"ghcr.io/datadog/dd-trace-dotnet/dd-lib-dotnet:{CommitSha}-musl";
+                    VerifyDockerImageExists(image);
+                }
+
+                static void VerifyDockerImageExists(string image)
+                {
+                    try
+                    {
+                        Logger.Information("Checking for presence of SSI image '{Image}'", image);
+                        DockerTasks.DockerManifest(
+                            s => s.SetCommand($"inspect")
+                                .SetProcessArgumentConfigurator(c => c.Add(image)));
+                        Logger.Information("SSI image '{Image}' exists", image);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new Exception($"Error verifying SSI artifacts: '{image}' could not be found. Ensure GitLab has successfully built and pushed the image", ex);
+                    }
+                }
+            });
+
     async Task ReplaceCommentInPullRequest(int prNumber, string title, string markdown)
     {
         try