mirrors/llvm-project
0
0
Fork 0
mirror of https://github.com/llvm/llvm-project.git synced 2025-04-16 01:06:32 +00:00
Code Issues Projects Releases Wiki Activity

[ci] Handle the case where all reported tests pass but the build is still a failure (#120264)

Browse Source 
In this build:
https://buildkite.com/llvm-project/github-pull-requests/builds/126961

The builds actually failed, probably because prerequisite of a test
suite failed to build.

However they still ran other tests and all those passed. This meant that
the test reports were green even though the build was red. On some level
this is technically correct, but it is very misleading in practice.

So I've also passed the build script's return code, as it was when we
entered the on exit handler, to the generator, so that when this happens
again, the report will draw the viewer's attention to the overall
failure. There will be a link in the report to the build's log file, so
the next step to investigate is clear.

It would be nice to say "tests failed and there was some other build
error", but we cannot tell what the non-zero return code was caused by.
Could be either.

The script handles the following situations now:
| Have Result Files? | Tests reported failed? | Return code | Report |

|--------------------|------------------------|-------------|-----------------------------------------------------------------------------|
| Yes | No | 0 | Success style report. |
| Yes | Yes | 0 | Shouldn't happen, but if it did, failure style report
showing the failures. |
| Yes | No | 1 | Failure style report, showing no failures but noting
that the build failed. |
| Yes | Yes | 1 | Failure style report, showing the test failures. |
| No | ? | 0 | No test report, success shown in the normal build
display. |
| No | ? | 1 | No test report, failure shown in the normal build
display. |
This commit is contained in:
David Spickett 2025-01-13 09:05:18 +00:00 committed by GitHub
parent b270525f73
commit 1b199d1990
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 94 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
.ci/generate_test_report.py
View File

@ -19,12 +19,13 @@ def junit_from_xml(xml):
class TestReports(unittest.TestCase):
def test_title_only(self):
self.assertEqual(_generate_report("Foo", []), ("", "success"))
self.assertEqual(_generate_report("Foo", 0, []), ("", "success"))
def test_no_tests_in_testsuite(self):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
@ -45,6 +46,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"Foo",
0,
[
junit_from_xml(
dedent(
@ -70,10 +72,51 @@ class TestReports(unittest.TestCase):
),
)
def test_no_failures_build_failed(self):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
"""\
<?xml version="1.0" encoding="UTF-8"?>
<testsuites time="0.00">
<testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
<testcase classname="Bar/test_1" name="test_1" time="0.00"/>
</testsuite>
</testsuites>"""
)
)
],
buildkite_info={
"BUILDKITE_ORGANIZATION_SLUG": "organization_slug",
"BUILDKITE_PIPELINE_SLUG": "pipeline_slug",
"BUILDKITE_BUILD_NUMBER": "build_number",
"BUILDKITE_JOB_ID": "job_id",
},
),
(
dedent(
"""\
# Foo
* 1 test passed
All tests passed but another part of the build **failed**.
[Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details."""
),
"error",
),
)
def test_report_single_file_single_testsuite(self):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
@ -166,6 +209,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"ABC and DEF",
1,
[
junit_from_xml(
dedent(
@ -198,6 +242,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"ABC and DEF",
1,
[
junit_from_xml(
dedent(
@ -238,6 +283,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
@ -272,6 +318,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
@ -312,6 +359,7 @@ class TestReports(unittest.TestCase):
self.assertEqual(
_generate_report(
"Foo",
1,
[
junit_from_xml(
dedent(
@ -351,12 +399,18 @@ class TestReports(unittest.TestCase):
# and output will not be.
def _generate_report(
title,
return_code,
junit_objects,
size_limit=1024 * 1024,
list_failures=True,
buildkite_info=None,
):
if not junit_objects:
# Note that we do not post an empty report, therefore we can ignore a
# non-zero return code in situations like this.
#
# If we were going to post a report, then yes, it would be misleading
# to say we succeeded when the final return code was non-zero.
return ("", "success")
failures = {}
@ -385,7 +439,11 @@ def _generate_report(
if not tests_run:
return ("", None)
style = "error" if tests_failed else "success"
style = "success"
# Either tests failed, or all tests passed but something failed to build.
if tests_failed or return_code != 0:
style = "error"
report = [f"# {title}", ""]
tests_passed = tests_run - tests_skipped - tests_failed
@ -400,17 +458,17 @@ def _generate_report(
if tests_failed:
report.append(f"* {tests_failed} {plural(tests_failed)} failed")
if not list_failures:
if buildkite_info is not None:
log_url = (
"https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"
"pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/"
"jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info)
)
download_text = f"[Download]({log_url})"
else:
download_text = "Download"
if buildkite_info is not None:
log_url = (
"https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"
"pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/"
"jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info)
)
download_text = f"[Download]({log_url})"
else:
download_text = "Download"
if not list_failures:
report.extend(
[
"",
@ -435,11 +493,23 @@ def _generate_report(
"</details>",
]
)
elif return_code != 0:
# No tests failed but the build was in a failed state. Bring this to the user's
# attention.
report.extend(
[
"",
"All tests passed but another part of the build **failed**.",
"",
f"{download_text} the build's log file to see the details.",
]
)
report = "\n".join(report)
if len(report.encode("utf-8")) > size_limit:
return _generate_report(
title,
return_code,
junit_objects,
size_limit,
list_failures=False,
@ -449,9 +519,10 @@ def _generate_report(
return report, style
def generate_report(title, junit_files, buildkite_info):
def generate_report(title, return_code, junit_files, buildkite_info):
return _generate_report(
title,
return_code,
[JUnitXml.fromfile(p) for p in junit_files],
buildkite_info=buildkite_info,
)
@ -463,6 +534,7 @@ if __name__ == "__main__":
"title", help="Title of the test report, without Markdown formatting."
)
parser.add_argument("context", help="Annotation context to write to.")
parser.add_argument("return_code", help="The build's return code.", type=int)
parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
args = parser.parse_args()
@ -477,7 +549,9 @@ if __name__ == "__main__":
if len(buildkite_info) != len(env_var_names):
buildkite_info = None
report, style = generate_report(args.title, args.junit_files, buildkite_info)
report, style = generate_report(
args.title, args.return_code, args.junit_files, buildkite_info
)
if report:
p = subprocess.Popen(

4
.ci/monolithic-linux.sh
View File

@ -29,6 +29,8 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
fi
function at-exit {
retcode=$?
mkdir -p artifacts
ccache --print-stats > artifacts/ccache_stats.txt
@ -37,7 +39,7 @@ function at-exit {
if command -v buildkite-agent 2>&1 >/dev/null
then
python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \
"linux-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
"linux-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
fi
}
trap at-exit EXIT

4
.ci/monolithic-windows.sh
View File

@ -28,6 +28,8 @@ fi
sccache --zero-stats
function at-exit {
retcode=$?
mkdir -p artifacts
sccache --show-stats >> artifacts/sccache_stats.txt
@ -36,7 +38,7 @@ function at-exit {
if command -v buildkite-agent 2>&1 >/dev/null
then
python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \
"windows-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
"windows-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
fi
}
trap at-exit EXIT