From 6011416d4d5823bdd94f11df9cbff46e4aa05e77 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Thu, 30 Oct 2025 16:40:23 -0400 Subject: [PATCH 1/8] Execute coverage logic (if enabled) prior to post action binary in xctestrunner This allows consumers in post action binary to act upon coverage results (if they are enabled) Signed-off-by: Maxwell Elliott --- .../ios_xctestrun_runner.template.sh | 165 +++++++++--------- 1 file changed, 79 insertions(+), 86 deletions(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 5527f544b..962d62d99 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -553,6 +553,85 @@ else 2>&1 | tee -i "$testlog" || test_exit_code=$? fi +if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then + profdata="$test_tmp_dir/$simulator_id/Coverage.profdata" + if [[ "$should_use_xcodebuild" == false ]]; then + profdata="$test_tmp_dir/coverage.profdata" + fi + + if [[ "${COLLECT_PROFDATA:-0}" == "1" && -f "$profdata" ]]; then + cp -R "$profdata" "$TEST_UNDECLARED_OUTPUTS_DIR" + fi + + if [[ "$should_use_xcodebuild" == false ]]; then + xcrun llvm-profdata merge "$profraw" --output "$profdata" + fi + + lcov_args=( + -instr-profile "$profdata" + -ignore-filename-regex='.*external/.+' + -path-equivalence=".,$PWD" + ) + has_binary=false + IFS=";" + arch=$(uname -m) + for binary in $TEST_BINARIES_FOR_LLVM_COV; do + if [[ "$has_binary" == false ]]; then + lcov_args+=("${binary}") + has_binary=true + if ! file "$binary" | grep -q "$arch"; then + arch=x86_64 + fi + else + lcov_args+=(-object "${binary}") + fi + + lcov_args+=("-arch=$arch") + done + + llvm_coverage_manifest="$COVERAGE_MANIFEST" + readonly provided_coverage_manifest="%(test_coverage_manifest)s" + if [[ -s "${provided_coverage_manifest:-}" ]]; then + llvm_coverage_manifest="$provided_coverage_manifest" + fi + + readonly error_file="$test_tmp_dir/llvm-cov-error.txt" + llvm_cov_status=0 + xcrun llvm-cov \ + export \ + -format lcov \ + "${lcov_args[@]}" \ + @"$llvm_coverage_manifest" \ + > "$COVERAGE_OUTPUT_FILE" \ + 2> "$error_file" \ + || llvm_cov_status=$? + + # Error ourselves if lcov outputs warnings, such as if we misconfigure + # something and the file path of one of the covered files doesn't exist + if [[ -s "$error_file" || "$llvm_cov_status" -ne 0 ]]; then + echo "error: while exporting coverage report" >&2 + cat "$error_file" >&2 + exit 1 + fi + + if [[ -n "${COVERAGE_PRODUCE_JSON:-}" ]]; then + llvm_cov_json_export_status=0 + xcrun llvm-cov \ + export \ + -format text \ + "${lcov_args[@]}" \ + @"$llvm_coverage_manifest" \ + > "$TEST_UNDECLARED_OUTPUTS_DIR/coverage.json" \ + 2> "$error_file" \ + || llvm_cov_json_export_status=$? + if [[ -s "$error_file" || "$llvm_cov_json_export_status" -ne 0 ]]; then + echo "error: while exporting json coverage report" >&2 + cat "$error_file" >&2 + exit 1 + fi + fi +fi + # Run a post-action binary, if provided. post_action_binary=%(post_action_binary)s post_action_determines_exit_code="%(post_action_determines_exit_code)s" @@ -591,15 +670,6 @@ if [[ "$reuse_simulator" == false ]]; then xcrun simctl delete "$simulator_id" fi -profdata="$test_tmp_dir/$simulator_id/Coverage.profdata" -if [[ "$should_use_xcodebuild" == false ]]; then - profdata="$test_tmp_dir/coverage.profdata" -fi - -if [[ "${COLLECT_PROFDATA:-0}" == "1" && -f "$profdata" ]]; then - cp -R "$profdata" "$TEST_UNDECLARED_OUTPUTS_DIR" -fi - if [[ "$post_action_determines_exit_code" == true ]]; then if [[ "$post_action_exit_code" -ne 0 ]]; then echo "error: post_action exited with '$post_action_exit_code'" >&2 @@ -666,83 +736,6 @@ then exit 1 fi -if [[ "${COVERAGE:-}" -ne 1 || "${APPLE_COVERAGE:-}" -ne 1 ]]; then - # Normal tests run without coverage - if [[ -f "${TEST_PREMATURE_EXIT_FILE:-}" ]]; then - rm -f "$TEST_PREMATURE_EXIT_FILE" - fi - - exit 0 -fi - -if [[ "$should_use_xcodebuild" == false ]]; then - xcrun llvm-profdata merge "$profraw" --output "$profdata" -fi - -lcov_args=( - -instr-profile "$profdata" - -ignore-filename-regex='.*external/.+' - -path-equivalence=".,$PWD" -) -has_binary=false -IFS=";" -arch=$(uname -m) -for binary in $TEST_BINARIES_FOR_LLVM_COV; do - if [[ "$has_binary" == false ]]; then - lcov_args+=("${binary}") - has_binary=true - if ! file "$binary" | grep -q "$arch"; then - arch=x86_64 - fi - else - lcov_args+=(-object "${binary}") - fi - - lcov_args+=("-arch=$arch") -done - -llvm_coverage_manifest="$COVERAGE_MANIFEST" -readonly provided_coverage_manifest="%(test_coverage_manifest)s" -if [[ -s "${provided_coverage_manifest:-}" ]]; then - llvm_coverage_manifest="$provided_coverage_manifest" -fi - -readonly error_file="$test_tmp_dir/llvm-cov-error.txt" -llvm_cov_status=0 -xcrun llvm-cov \ - export \ - -format lcov \ - "${lcov_args[@]}" \ - @"$llvm_coverage_manifest" \ - > "$COVERAGE_OUTPUT_FILE" \ - 2> "$error_file" \ - || llvm_cov_status=$? - -# Error ourselves if lcov outputs warnings, such as if we misconfigure -# something and the file path of one of the covered files doesn't exist -if [[ -s "$error_file" || "$llvm_cov_status" -ne 0 ]]; then - echo "error: while exporting coverage report" >&2 - cat "$error_file" >&2 - exit 1 -fi - -if [[ -n "${COVERAGE_PRODUCE_JSON:-}" ]]; then - llvm_cov_json_export_status=0 - xcrun llvm-cov \ - export \ - -format text \ - "${lcov_args[@]}" \ - @"$llvm_coverage_manifest" \ - > "$TEST_UNDECLARED_OUTPUTS_DIR/coverage.json" \ - 2> "$error_file" \ - || llvm_cov_json_export_status=$? - if [[ -s "$error_file" || "$llvm_cov_json_export_status" -ne 0 ]]; then - echo "error: while exporting json coverage report" >&2 - cat "$error_file" >&2 - exit 1 - fi -fi - if [[ -f "${TEST_PREMATURE_EXIT_FILE:-}" ]]; then rm -f "$TEST_PREMATURE_EXIT_FILE" fi From 30f151350ba3b37a0506c11d159f27ea39a7a353 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 31 Oct 2025 15:39:26 -0400 Subject: [PATCH 2/8] feedback Signed-off-by: Maxwell Elliott --- apple/testing/default_runner/ios_xctestrun_runner.template.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 962d62d99..cf4d9f1c4 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -627,7 +627,6 @@ if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then if [[ -s "$error_file" || "$llvm_cov_json_export_status" -ne 0 ]]; then echo "error: while exporting json coverage report" >&2 cat "$error_file" >&2 - exit 1 fi fi fi @@ -641,6 +640,8 @@ if [[ -n "${result_bundle_path:-}" ]]; then TEST_LOG_FILE="$testlog" \ SIMULATOR_UDID="$simulator_id" \ TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \ + LLVM_COV_STATUS="${llvm_cov_status:-0}" \ + LLVM_COV_JSON_EXPORT_STATUS="${llvm_cov_json_export_status:-0}" \ "$post_action_binary" || post_action_exit_code=$? else TEST_EXIT_CODE=$test_exit_code \ From 1a1dfd7d07a2a2106c27726f68e8c7d55d8dbbca Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 31 Oct 2025 15:52:02 -0400 Subject: [PATCH 3/8] Update apple/testing/default_runner/ios_xctestrun_runner.template.sh Co-authored-by: Brentley Jones --- apple/testing/default_runner/ios_xctestrun_runner.template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index cf4d9f1c4..68c5d3115 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -640,7 +640,7 @@ if [[ -n "${result_bundle_path:-}" ]]; then TEST_LOG_FILE="$testlog" \ SIMULATOR_UDID="$simulator_id" \ TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \ - LLVM_COV_STATUS="${llvm_cov_status:-0}" \ + LLVM_COV_EXIT_CODE="${llvm_cov_status:-0}" \ LLVM_COV_JSON_EXPORT_STATUS="${llvm_cov_json_export_status:-0}" \ "$post_action_binary" || post_action_exit_code=$? else From 7b050367f5fdf354436389637bd9feb6ce91ce05 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 31 Oct 2025 15:52:19 -0400 Subject: [PATCH 4/8] Update apple/testing/default_runner/ios_xctestrun_runner.template.sh Co-authored-by: Brentley Jones --- apple/testing/default_runner/ios_xctestrun_runner.template.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 68c5d3115..6f1062f43 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -641,7 +641,7 @@ if [[ -n "${result_bundle_path:-}" ]]; then SIMULATOR_UDID="$simulator_id" \ TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \ LLVM_COV_EXIT_CODE="${llvm_cov_status:-0}" \ - LLVM_COV_JSON_EXPORT_STATUS="${llvm_cov_json_export_status:-0}" \ + LLVM_COV_JSON_EXPORT_EXIT_CODE="${llvm_cov_json_export_status:-0}" \ "$post_action_binary" || post_action_exit_code=$? else TEST_EXIT_CODE=$test_exit_code \ From 2882c66be13fd25cc1e5d64bb6890f9390b1413c Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 31 Oct 2025 15:54:24 -0400 Subject: [PATCH 5/8] match existing behaviour Signed-off-by: Maxwell Elliott --- .../default_runner/ios_xctestrun_runner.template.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 6f1062f43..e367fd2f8 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -611,7 +611,6 @@ if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then if [[ -s "$error_file" || "$llvm_cov_status" -ne 0 ]]; then echo "error: while exporting coverage report" >&2 cat "$error_file" >&2 - exit 1 fi if [[ -n "${COVERAGE_PRODUCE_JSON:-}" ]]; then @@ -737,6 +736,16 @@ then exit 1 fi +if [[ "$llvm_cov_status" -ne 0 ]]; then + echo "error: while exporting coverage report" >&2 + exit 1 +fi + +if [[ "$llvm_cov_json_export_status" -ne 0 ]]; then + echo "error: while exporting json coverage report" >&2 + exit 1 +fi + if [[ -f "${TEST_PREMATURE_EXIT_FILE:-}" ]]; then rm -f "$TEST_PREMATURE_EXIT_FILE" fi From ccf35eec259a6aa9a3bb0716b5c1b4291d068da5 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Thu, 13 Nov 2025 12:46:49 -0500 Subject: [PATCH 6/8] more updates --- .../default_runner/ios_xctestrun_runner.template.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index e367fd2f8..2e75c1371 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -553,6 +553,8 @@ else 2>&1 | tee -i "$testlog" || test_exit_code=$? fi +llvm_cov_status=0 +llvm_cov_json_export_status=0 if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then profdata="$test_tmp_dir/$simulator_id/Coverage.profdata" if [[ "$should_use_xcodebuild" == false ]]; then @@ -736,13 +738,15 @@ then exit 1 fi -if [[ "$llvm_cov_status" -ne 0 ]]; then +if [[ -s "$error_file" || "$llvm_cov_status" -ne 0 ]]; then echo "error: while exporting coverage report" >&2 + cat "$error_file" >&2 exit 1 fi -if [[ "$llvm_cov_json_export_status" -ne 0 ]]; then +if [[ -s "$error_file" || "$llvm_cov_json_export_status" -ne 0 ]]; then echo "error: while exporting json coverage report" >&2 + cat "$error_file" >&2 exit 1 fi From 89d363c159ada98b1258ef598e65b6648e784f15 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Thu, 13 Nov 2025 19:57:47 -0500 Subject: [PATCH 7/8] more updates --- .../ios_xctestrun_runner.template.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 2e75c1371..285c8e8b3 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -597,7 +597,6 @@ if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then llvm_coverage_manifest="$provided_coverage_manifest" fi - readonly error_file="$test_tmp_dir/llvm-cov-error.txt" llvm_cov_status=0 xcrun llvm-cov \ export \ @@ -738,16 +737,14 @@ then exit 1 fi -if [[ -s "$error_file" || "$llvm_cov_status" -ne 0 ]]; then - echo "error: while exporting coverage report" >&2 - cat "$error_file" >&2 - exit 1 +if [[ "$llvm_cov_status" -ne 0 ]]; then + echo "failure: exporting coverage report failed" >&2 + exit "$llvm_cov_status" fi -if [[ -s "$error_file" || "$llvm_cov_json_export_status" -ne 0 ]]; then - echo "error: while exporting json coverage report" >&2 - cat "$error_file" >&2 - exit 1 +if [[ "$llvm_cov_json_export_status" -ne 0 ]]; then + echo "failure: exporting json coverage report failed" >&2 + exit "$llvm_cov_json_export_status" fi if [[ -f "${TEST_PREMATURE_EXIT_FILE:-}" ]]; then From 16c34fd500121edc45d60c2937b25e3a6bfea7e5 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Thu, 13 Nov 2025 21:04:04 -0500 Subject: [PATCH 8/8] ci --- .../default_runner/ios_xctestrun_runner.template.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh index 285c8e8b3..8de8633d1 100755 --- a/apple/testing/default_runner/ios_xctestrun_runner.template.sh +++ b/apple/testing/default_runner/ios_xctestrun_runner.template.sh @@ -553,8 +553,6 @@ else 2>&1 | tee -i "$testlog" || test_exit_code=$? fi -llvm_cov_status=0 -llvm_cov_json_export_status=0 if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then profdata="$test_tmp_dir/$simulator_id/Coverage.profdata" if [[ "$should_use_xcodebuild" == false ]]; then @@ -597,6 +595,7 @@ if [[ "${COVERAGE:-}" -eq 1 || "${APPLE_COVERAGE:-}" -eq 1 ]]; then llvm_coverage_manifest="$provided_coverage_manifest" fi + readonly error_file="$test_tmp_dir/llvm-cov-error.txt" llvm_cov_status=0 xcrun llvm-cov \ export \ @@ -737,13 +736,13 @@ then exit 1 fi -if [[ "$llvm_cov_status" -ne 0 ]]; then - echo "failure: exporting coverage report failed" >&2 +if [[ "${llvm_cov_status:-0}" -ne 0 ]]; then + echo "error: exporting coverage report failed" >&2 exit "$llvm_cov_status" fi -if [[ "$llvm_cov_json_export_status" -ne 0 ]]; then - echo "failure: exporting json coverage report failed" >&2 +if [[ "${llvm_cov_json_export_status:-0}" -ne 0 ]]; then + echo "error: exporting json coverage report failed" >&2 exit "$llvm_cov_json_export_status" fi