Merge remote-tracking branch 'origin/master' into zach.montoya/metrics-optimizations

Author: zacharycmontoya

.azure-pipelines/ultimate-pipeline.yml

@@ -2777,6 +2777,7 @@ stages:
         useNativeSdkVersion: $(useNativeSdkVersion)
         command: "BuildProfilerSamples"
         apiKey: $(DD_LOGGER_DD_API_KEY)
+        retryCountForRunCommand: 3
 
     - template: steps/run-in-docker.yml
       parameters:
@@ -2785,6 +2786,7 @@ stages:
         command: "BuildAndRunProfilerCpuLimitTests"
         extraArgs: "--cpus 2 --env CONTAINER_CPUS=1"
         apiKey: $(DD_LOGGER_DD_API_KEY)
+        retryCountForRunCommand: 3
 
     - template: steps/run-in-docker.yml
       parameters:
@@ -2793,6 +2795,7 @@ stages:
         command: "BuildAndRunProfilerCpuLimitTests"
         extraArgs: "--cpus 0.5 --env CONTAINER_CPUS=0.5"
         apiKey: $(DD_LOGGER_DD_API_KEY)
+        retryCountForRunCommand: 3
 
     - script: |
         docker-compose -f docker-compose.yml -p $(DockerComposeProjectName) \
@@ -4238,6 +4241,10 @@ stages:
               echo "Label 'docker_image_artifacts' not found in PR — skipping Docker image build"
               exit 0
             fi
+            if ! [[ $BRANCH_NAME =~ ^[A-Za-z0-9._/-]+$ ]]; then
+              echo "Exiting - Manages branches only if they contain letters, numbers and the following special characters: . - _ /"
+              exit 1
+            fi
           fi
 
           echo "Creating dispatch event for https://github.com/DataDog/dd-trace-dotnet/actions/workflows/create-system-test-docker-base-images.yml with ref=$REF and azdo_build_id=$(Build.BuildId) and is_release_version=False"

.github/chainguard/gitlab.github-access.read-contents.sts.yaml

@@ -0,0 +1,6 @@
+issuer: https://gitlab.ddbuild.io
+
+subject_pattern: "project_path:DataDog/apm-reliability/dd-trace-dotnet:ref_type:(branch|tag):ref:.*"
+
+permissions:
+  contents: read

.gitlab/benchmarks/macrobenchmarks.yml

@@ -467,6 +467,13 @@ profiler_cpu_timer_create-arm64:
       - platform/artifacts/
     expire_in: 3 months
   variables:
+    AWS_REGION: "us-east-1"
+    
+    # Branch containing 1. scripts to launch Windows benchmarks on ephemeral 
+    # instances (to be used by GitLab CI runners) and 2. scripts to run Windows 
+    # benchmarks (to be used by the ephemeral instances).
+    BP_INFRA_BENCHMARKING_PLATFORM_BRANCH: "dd-trace-dotnet/macro"
+
     # Whether to cleanup ephemeral instances after benchmarks are run
     CLEANUP: "true"
 
@@ -489,14 +496,17 @@ profiler_cpu_timer_create-arm64:
   script:
     - source build-id.txt
     - echo "Building for the following build https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=$buildId&view=results"
-    - export BP_INFRA_BENCHMARKING_PLATFORM_BRANCH=dd-trace-dotnet/macro
     - git clone --branch $BP_INFRA_BENCHMARKING_PLATFORM_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
     - ./ephemeral-infra/run-windows-benchmarks.sh
   after_script:
     - |
-      bp-infra cleanup --provision ./platform/ephemeral-infra/provisions/macrobenchmark-ephemeral-instance.yaml \
-        --region "${AWS_REGION}" \
-        --bypass-stack-destroy
+      if [ "$CLEANUP" == "true" ]; then
+        bp-infra cleanup --provision ./platform/ephemeral-infra/provisions/macrobenchmark-ephemeral-instance.yaml \
+          --region "${AWS_REGION}" \
+          --bypass-stack-destroy
+      else
+        echo "'CLEANUP' is set to 'false'. Will not cleanup."
+      fi
 
 baseline-win:
   extends: .benchmarks-win

.gitlab/benchmarks/microbenchmarks.yml

@@ -1,101 +1,129 @@
-.setup:
-  script:
-    - mkdir -p ~/.aws
-    - /app/bp-infra/tools/fetch-ssm-parameter.sh $AWS_EPHEMERAL_INFRA_PROFILE_SSM_PARAMETER > ~/.aws/config || exit $?
-    - export AWS_PROFILE=ephemeral-infra-ci
-    - export BP_INFRA_KEY_PAIR_NAME=$(cat ~/.aws/key-pair-name.txt)
-    - export BP_INFRA_KEY_PAIR_PRIVATE_KEY_PATH=~/.aws/key-pair-private-key.pem
+.dd-octo-sts-setup:
+  before_script:
+    - |
+      set +e
+      echo "Attempting to retrieve a GitHub token for scope '$DDOCTOSTS_SCOPE' with policy '$DDOCTOSTS_POLICY' with dd-octo-sts..."
+      error_output=$({ dd-octo-sts token --scope $DDOCTOSTS_SCOPE --policy $DDOCTOSTS_POLICY > "/tmp/github-token"; } 2>&1)
+      exit_code=$?
+      if [ $exit_code -ne 0 ]; then
+        echo "ERROR: Failed to retrieve GitHub token."
+        echo "Original error: $error_output"
+        echo "Continuing execution anyway..."
+      fi
+      set -e
 
 stages:
-  - infra-update
+  - build
   - benchmarks
 
-update-bp-infra:
-  stage: infra-update
+build-dd-trace-dotnet-microbenchmarks-ami:
+  stage: build
+  tags: ["arch:amd64"]
   timeout: 3h
-  tags: ["arch:amd64"]  
   allow_failure: true
-  # Image created in the following job https://gitlab.ddbuild.io/DataDog/benchmarking-platform-tools/-/jobs/869830045
+  when: manual
   image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:dd-trace-dotnet-micro
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  variables:
+    # Allows ephemeral instances to read content from dd-trace-dotnet
+    # This is not strictly necessary in the current AMI build
+    DDOCTOSTS_SCOPE: "DataDog/dd-trace-dotnet"
+    DDOCTOSTS_POLICY: "gitlab.github-access.read-contents"
+
+    AWS_REGION: "us-east-1"
+
+    # TODO: Point to dd-trace-dotnet/micro after the branch below is merged
+    # to it
+    # Branch containing a provision for building the AMI
+    BP_INFRA_BENCHMARKING_PLATFORM_BRANCH: "augusto/dd-trace-dotnet-micro-bp-ui"
+
+    PROVISION_FILE: "platform/ephemeral-infra/ami.yaml"
+
+    # Where AMI creation artifacts will be stored
+    BP_INFRA_ARTIFACTS_BUCKET_NAME: "windows-benchmarking-results-us-east-1"
 
+    # Whether to cleanup instances after building the AMI, since the AMI is 
+    # based on an instance that is created in this job
+    CLEANUP: "true"
+  before_script:
+    - !reference [.dd-octo-sts-setup, before_script]
   script:
-    - git clone --branch dd-trace-dotnet/micro https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform
-    - mkdir -p ~/.aws
-    - /app/bp-infra/tools/fetch-ssm-parameter.sh $AWS_EPHEMERAL_INFRA_PROFILE_SSM_PARAMETER >> ~/.aws/config || exit $?
-    - aws ssm get-parameter --region "$AWS_REGION" --name "ci.${CI_PROJECT_NAME}.ephemeral-infra-ci.windows-benchmarking-key-pair-name" --with-decryption --query "Parameter.Value" --out text >> ~/.aws/key-pair-name.txt
-    - aws ssm get-parameter --region "$AWS_REGION" --name "ci.${CI_PROJECT_NAME}.ephemeral-infra-ci.windows-benchmarking-key-private-key" --with-decryption --query "Parameter.Value" --out text >> ~/.aws/key-pair-private-key.pem
-    - export AWS_PROFILE=ephemeral-infra-ci
-    - export BP_INFRA_KEY_PAIR_NAME=$(cat ~/.aws/key-pair-name.txt)
-    - export BP_INFRA_KEY_PAIR_PRIVATE_KEY_PATH=~/.aws/key-pair-private-key.pem
-    - bp-infra launch --provision ./platform/ephemeral-infra/base-instance.yaml --region "${AWS_REGION}" --bypass-stack-destroy
+    - git clone --branch $BP_INFRA_BENCHMARKING_PLATFORM_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform
+    - echo "GITHUB_TOKEN=$(cat /tmp/github-token)" > .env
+    - CLEANUP_ARG=$([[ "$CLEANUP" == "false" ]] && echo "--no-cleanup" || echo "")
+    - |
+      bp-infra launch --region "${AWS_REGION}" --os "windows" \
+        --provision "${PROVISION_FILE}" \
+        --bypass-stack-destroy \
+        --env .env \
+        $CLEANUP_ARG
   after_script:
-    - !reference [.setup, script]
+    # Makes sure the instance is cleaned up.
+    # Note: This does not clean up the created AMI.
     - |
-      bp-infra cleanup --provision ./platform/ephemeral-infra/base-instance.yaml \
-        --region "${AWS_REGION}" \
-        --bypass-stack-destroy
-    
-  rules:
-    - when: manual
-  variables:
-    AWS_REGION: "us-east-1"
-    CLEANUP: "false"
-    AWS_EPHEMERAL_INFRA_PROFILE_SSM_PARAMETER: "ci.dd-trace-dotnet.ephemeral-infra-ci.dd-trace-dotnet-profile"
-    AWS_EPHEMERAL_INFRA_PROFILE_NAME: "ephemeral-infra-ci"
-    AWS_EPHEMERAL_INFRA_ARTIFACTS_BUCKET_URI: "s3://windows-benchmarking-results/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_JOB_ID"
-    AWS_EPHEMERAL_INFRA_REGION: "us-east-1"
+      if [ "$CLEANUP" == "true" ]; then
+        bp-infra cleanup --region "${AWS_REGION}" --os "windows" \
+          --provision "${PROVISION_FILE}" \
+          --bypass-stack-destroy
+      else 
+        echo "'CLEANUP' is set to 'false'. Will not cleanup."
+      fi
 
 run-benchmarks:
   stage: benchmarks
   tags: ["arch:amd64"]
   timeout: 2h
   # Image created in the following job https://gitlab.ddbuild.io/DataDog/benchmarking-platform-tools/-/jobs/869830045
   image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:dd-trace-dotnet-micro
-
-  script:
-    - git clone --branch dd-trace-dotnet/micro https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform
-    - AWS_REGION=${AWS_REGION} ./platform/steps/launch-instance.sh
-  after_script:
-    - |
-      bp-infra cleanup --provision ./platform/ephemeral-infra/ephemeral-instance-main.yaml \
-        --region "${AWS_REGION}" \
-        --bypass-stack-destroy
-    - ./platform/steps/post-pr-comment.sh
-    # Temporarily commented out pending issue resolution with sending files to backend
-    # - ./platform/steps/upload-to-bp-ui.sh
-    
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   rules:
     - when: on_success
-  variables:
-    AWS_REGION: "us-east-1"
-
-upload-to-bp-ui:
-  stage: benchmarks
-  tags: ["arch:amd64"]
-  timeout: 1h
-  # Image created in the following job https://gitlab.ddbuild.io/DataDog/benchmarking-platform-tools/-/jobs/869830045
-  image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:dotnet-microbenchmarks
-
-  script:
-    - git clone --branch fayssal/test-micro-delivery https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform
-    # - ./platform/steps/launch-instance.sh
-    # - ./platform/steps/post-pr-comment.sh
-    # Temporarely commented out pending issue resolution with sending files to backend
-    - ./platform/steps/upload-to-bp-ui.sh
-
   artifacts:
     name: "artifacts"
     when: always
     paths:
-      - candidate-results/
+      - platform/artifacts
     expire_in: 3 months
-    
-  rules:
-    - when: manual
   variables:
+    # Allows ephemeral instances to read content from benchmarking-platform
+    DDOCTOSTS_SCOPE: "DataDog/benchmarking-platform"
+    DDOCTOSTS_POLICY: "gitlab.github-access.read-contents"
+
     AWS_REGION: "us-east-1"
-    CLEANUP: "false"
-    AWS_EPHEMERAL_INFRA_PROFILE_SSM_PARAMETER: "ci.dd-trace-dotnet.ephemeral-infra-ci.dd-trace-dotnet-profile"
-    AWS_EPHEMERAL_INFRA_PROFILE_NAME: "ephemeral-infra-ci"
-    AWS_EPHEMERAL_INFRA_ARTIFACTS_BUCKET_URI: "s3://windows-benchmarking-results/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_JOB_ID"
-    AWS_EPHEMERAL_INFRA_REGION: "us-east-1"
+
+    # TODO: Point to dd-trace-dotnet/micro after the branch below is merged
+    # to it
+    # Branch containing 1. scripts to launch Windows benchmarks on ephemeral 
+    # instances (to be used by GitLab CI runners) and 2. scripts to run Windows 
+    # benchmarks (to be used by the ephemeral instances).
+    BP_INFRA_BENCHMARKING_PLATFORM_BRANCH: "augusto/dd-trace-dotnet-micro-bp-ui"
+
+    # Where benchmarking results will be stored
+    BP_INFRA_ARTIFACTS_BUCKET_NAME: "windows-benchmarking-results-us-east-1"
+
+    # Whether to cleanup ephemeral instances after benchmarks are run
+    CLEANUP: "true"
+
+    # Where to look for benchmarking artifacts for uploading to the BP UI
+    ARTIFACTS_DIR: "platform/artifacts"
+  before_script:
+    - !reference [.dd-octo-sts-setup, before_script]
+  script:
+    - export GITHUB_TOKEN=$(cat /tmp/github-token)
+    - git clone --branch $BP_INFRA_BENCHMARKING_PLATFORM_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform
+    - ./platform/steps/run-windows-benchmarks.sh
+  after_script:
+    - |
+      if [ "$CLEANUP" == "true" ]; then
+        bp-infra cleanup --provision ./platform/ephemeral-infra/instance.yaml \
+          --region "${AWS_REGION}" \
+          --bypass-stack-destroy
+      else
+        echo "'CLEANUP' is set to 'false'. Will not cleanup."
+      fi
+    - ./platform/steps/post-pr-comment.sh
+    - ./platform/steps/upload-to-bp-ui.sh

AGENTS.md

@@ -293,6 +293,9 @@ The tracer runs in-process with customer applications and must have minimal perf
 - `docs/development/AzureFunctions.md` — Azure Functions integration
 - `docs/development/Serverless.md` — Serverless instrumentation
 - `docs/development/UpdatingTheSdk.md` — SDK updates
+
+**CI & Testing:**
+- `docs/development/CI/TroubleshootingCIFailures.md` — Investigating build/test failures in Azure DevOps
 - `docs/development/CI/RunSmokeTestsLocally.md` — Running smoke tests locally
 
 ## Security & Configuration

docs/development/AzureFunctions.md

@@ -42,7 +42,7 @@ To correctly flow and parent the state across each request, our integration does
 - In-process instrumentation of `FunctionInvocationMiddleware` in `func.exe`.
   - This is the same integration we use for normal in-process functions. We need to keep the span generated here as it represents the "real" latencies seen by customers calling the function. This is only created for HTTP requests.
 - In-process instrumentation of `FunctionExecutor` in `func.exe`.
-  - This is the same integration point as for in-process functions, but it serves a slightly different purpose. We don't want to create a span here (this function invocation doesn't represent anything useful), but we _do_ want to enrich the top-level HTTP span with more details about the function invocation. 
+  - This is the same integration point as for in-process functions, but it serves a slightly different purpose. We don't want to create a span here (this function invocation doesn't represent anything useful), but we _do_ want to enrich the top-level HTTP span with more details about the function invocation.
   - The "function" being executed here is the "fake" in-process function which mirrors the "real" out-of-process function. Consequently, we don't have real type names, and the function name is prefixed with `Functions.`, e.g. `Functions.MyAppTrigger` instead of `MyAppTrigger`
 - Instrumentation of `GrpcMessageConversionExtensions.ToRpcHttp()` in `func.exe`.
   - This is a new integration, which hooks into the method that converts the incoming HTTP request into a GRPC message.
@@ -57,7 +57,7 @@ For an HTTP trigger function, the result is something like this:
 
 ```mermaid
 gantt
-    title HTTP span 
+    title HTTP span
     dateFormat  X
     axisFormat %s
     todayMarker off
@@ -70,7 +70,7 @@ For a timer trigger, there's a single span
 
 ```mermaid
 gantt
-    title HTTP span 
+    title HTTP span
     dateFormat  X
     axisFormat %s
     todayMarker off
@@ -95,6 +95,36 @@ Whether a Functions app uses this new mode is subtle:
 
 and the project will have a reference to [Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore](https://www.nuget.org/packages/Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore/) package.
 
+## Local Development
+
+### Building the Datadog.AzureFunctions NuGet Package
+
+For rapid iteration when testing changes to `Datadog.Trace` in Azure Functions projects, use the `Build-AzureFunctionsNuget.ps1` helper script:
+
+```powershell
+# Build using existing bundle
+.\tracer\tools\Build-AzureFunctionsNuget.ps1
+
+# Download bundle from a specific Azure DevOps build first
+.\tracer\tools\Build-AzureFunctionsNuget.ps1 -BuildId 12345
+
+# Build and copy package to a specific location
+.\tracer\tools\Build-AzureFunctionsNuget.ps1 -CopyTo 'D:\temp\nuget'
+
+# Use -Verbose for detailed progress
+.\tracer\tools\Build-AzureFunctionsNuget.ps1 -Verbose
+```
+
+This script:
+1. Cleans up previous builds
+2. Removes the `Datadog.AzureFunctions` package from the local NuGet cache
+3. Optionally downloads `Datadog.Trace.Bundle` from a specific build
+4. Builds `Datadog.Trace` and publishes to the bundle folder (`net6.0` and `net461` targets)
+5. Builds the `Datadog.AzureFunctions` NuGet package
+6. Optionally copies the package to a specified path
+
+See `Get-Help .\tracer\tools\Build-AzureFunctionsNuget.ps1 -Full` for complete documentation.
+
 ## Debugging
 
 To debug Azure Functions locally ensure that you have the following: