From 889b3c9487d114b9d082e9552599c8a8a8ccc660 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Wed, 13 Nov 2024 09:19:10 +0000 Subject: [PATCH] Reland "[ci] New script to generate test reports as Buildkite Annotations (#113447)" This reverts commit 8a1ca6cad9cd0e972c322910cdfbbe9552c6c7ca. I have fixed 2 things: * The report is now sent by stdin so we do not hit the limit on the size of command line arguments. * The report is limited to 1MB in size and if we exceed that we fall back to listing only the totals with a note telling you to check the full log. --- .ci/generate_test_report.py | 424 ++++++++++++++++++++++++++++++++++++ .ci/monolithic-linux.sh | 10 +- .ci/monolithic-windows.sh | 10 +- .ci/requirements.txt | 1 + 4 files changed, 441 insertions(+), 4 deletions(-) create mode 100644 .ci/generate_test_report.py create mode 100644 .ci/requirements.txt diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py new file mode 100644 index 000000000000..b2ab81ae4e01 --- /dev/null +++ b/.ci/generate_test_report.py @@ -0,0 +1,424 @@ +# Script to parse many JUnit XML result files and send a report to the buildkite +# agent as an annotation. +# +# To run the unittests: +# python3 -m unittest discover -p generate_test_report.py + +import argparse +import subprocess +import unittest +from io import StringIO +from junitparser import JUnitXml, Failure +from textwrap import dedent + + +def junit_from_xml(xml): + return JUnitXml.fromfile(StringIO(xml)) + + +class TestReports(unittest.TestCase): + def test_title_only(self): + self.assertEqual(_generate_report("Foo", []), ("", None)) + + def test_no_tests_in_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + """ + ) + ) + ], + ), + ("", None), + ) + + def test_no_failures(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed""" + ), + "success", + ), + ) + + def test_report_single_file_single_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed Tests + (click to see output) + + ### Bar +
+ Bar/test_3/test_3 + + ``` + Output goes here + ``` +
+
+ Bar/test_4/test_4 + + ``` + Other output goes here + ``` +
""" + ), + "error", + ), + ) + + MULTI_SUITE_OUTPUT = ( + dedent( + """\ + # ABC and DEF + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed Tests + (click to see output) + + ### ABC +
+ ABC/test_2/test_2 + + ``` + ABC/test_2 output goes here + ``` +
+ + ### DEF +
+ DEF/test_2/test_2 + + ``` + DEF/test_2 output goes here + ``` +
""" + ), + "error", + ) + + def test_report_single_file_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + def test_report_multiple_files_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + """ + ) + ), + junit_from_xml( + dedent( + """\ + + + + + + + + + + + """ + ) + ), + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + def test_report_dont_list_failures(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + """ + ) + ) + ], + list_failures=False, + ), + ( + dedent( + """\ + # Foo + + * 1 test failed + + Failed tests and their output was too large to report. Download the build's log file to see the details.""" + ), + "error", + ), + ) + + def test_report_size_limit(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + """ + ) + ) + ], + size_limit=128, + ), + ( + dedent( + """\ + # Foo + + * 1 test failed + + Failed tests and their output was too large to report. Download the build's log file to see the details.""" + ), + "error", + ), + ) + + +# Set size_limit to limit the byte size of the report. The default is 1MB as this +# is the most that can be put into an annotation. If the generated report exceeds +# this limit and failures are listed, it will be generated again without failures +# listed. This minimal report will always fit into an annotation. +# If include failures is False, total number of test will be reported but their names +# and output will not be. +def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures=True): + style = None + + if not junit_objects: + return ("", style) + + failures = {} + tests_run = 0 + tests_skipped = 0 + tests_failed = 0 + + for results in junit_objects: + for testsuite in results: + tests_run += testsuite.tests + tests_skipped += testsuite.skipped + tests_failed += testsuite.failures + + for test in testsuite: + if ( + not test.is_passed + and test.result + and isinstance(test.result[0], Failure) + ): + if failures.get(testsuite.name) is None: + failures[testsuite.name] = [] + failures[testsuite.name].append( + (test.classname + "/" + test.name, test.result[0].text) + ) + + if not tests_run: + return ("", style) + + style = "error" if tests_failed else "success" + report = [f"# {title}", ""] + + tests_passed = tests_run - tests_skipped - tests_failed + + def plural(num_tests): + return "test" if num_tests == 1 else "tests" + + if tests_passed: + report.append(f"* {tests_passed} {plural(tests_passed)} passed") + if tests_skipped: + report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped") + if tests_failed: + report.append(f"* {tests_failed} {plural(tests_failed)} failed") + + if not list_failures: + report.extend( + [ + "", + "Failed tests and their output was too large to report. " + "Download the build's log file to see the details.", + ] + ) + elif failures: + report.extend(["", "## Failed Tests", "(click to see output)"]) + + for testsuite_name, failures in failures.items(): + report.extend(["", f"### {testsuite_name}"]) + for name, output in failures: + report.extend( + [ + "
", + f"{name}", + "", + "```", + output, + "```", + "
", + ] + ) + + report = "\n".join(report) + if len(report.encode("utf-8")) > size_limit: + return _generate_report(title, junit_objects, size_limit, list_failures=False) + + return report, style + + +def generate_report(title, junit_files): + return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "title", help="Title of the test report, without Markdown formatting." + ) + parser.add_argument("context", help="Annotation context to write to.") + parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") + args = parser.parse_args() + + report, style = generate_report(args.title, args.junit_files) + + p = subprocess.Popen( + [ + "buildkite-agent", + "annotate", + "--context", + args.context, + "--style", + style, + ], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + # The report can be larger than the buffer for command arguments so we send + # it over stdin instead. + _, err = p.communicate(input=report) + if p.returncode: + raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}") diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index 17ea51c08faf..a4aeea7a16ad 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -28,11 +28,16 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then ccache --clear fi -function show-stats { +function at-exit { mkdir -p artifacts ccache --print-stats > artifacts/ccache_stats.txt + + # If building fails there will be no results files. + shopt -s nullglob + python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \ + "linux-x64-test-results" "${BUILD_DIR}"/test-results.*.xml } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" @@ -42,6 +47,7 @@ lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-outpu echo "--- cmake" pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt +pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 9ec44c22442d..4ead122212f4 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -27,17 +27,23 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then fi sccache --zero-stats -function show-stats { +function at-exit { mkdir -p artifacts sccache --show-stats >> artifacts/sccache_stats.txt + + # If building fails there will be no results files. + shopt -s nullglob + python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \ + "windows-x64-test-results" "${BUILD_DIR}"/test-results.*.xml } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" echo "--- cmake" pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt +pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research # on fixing a build reliability issue on the build server, please diff --git a/.ci/requirements.txt b/.ci/requirements.txt new file mode 100644 index 000000000000..ad63858c9fdc --- /dev/null +++ b/.ci/requirements.txt @@ -0,0 +1 @@ +junitparser==3.2.0