diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py
new file mode 100644
index 000000000000..f2ae116ace99
--- /dev/null
+++ b/.ci/generate_test_report.py
@@ -0,0 +1,328 @@
+# Script to parse many JUnit XML result files and send a report to the buildkite
+# agent as an annotation.
+#
+# To run the unittests:
+# python3 -m unittest discover -p generate_test_report.py
+
+import argparse
+import unittest
+from io import StringIO
+from junitparser import JUnitXml, Failure
+from textwrap import dedent
+from subprocess import check_call
+
+
+def junit_from_xml(xml):
+ return JUnitXml.fromfile(StringIO(xml))
+
+
+class TestReports(unittest.TestCase):
+ def test_title_only(self):
+ self.assertEqual(_generate_report("Foo", []), ("", None))
+
+ def test_no_tests_in_testsuite(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+ """
+ )
+ )
+ ],
+ ),
+ ("", None),
+ )
+
+ def test_no_failures(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed"""
+ ),
+ "success",
+ ),
+ )
+
+ def test_report_single_file_single_testsuite(self):
+ self.assertEqual(
+ _generate_report(
+ "Foo",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+ * 1 test skipped
+ * 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### Bar
+
+ Bar/test_3/test_3
+
+ ```
+ Output goes here
+ ```
+
+
+ Bar/test_4/test_4
+
+ ```
+ Other output goes here
+ ```
+ """
+ ),
+ "error",
+ ),
+ )
+
+ MULTI_SUITE_OUTPUT = (
+ dedent(
+ """\
+ # ABC and DEF
+
+ * 1 test passed
+ * 1 test skipped
+ * 2 tests failed
+
+ ## Failed tests
+ (click to see output)
+
+ ### ABC
+
+ ABC/test_2/test_2
+
+ ```
+ ABC/test_2 output goes here
+ ```
+
+
+ ### DEF
+
+ DEF/test_2/test_2
+
+ ```
+ DEF/test_2 output goes here
+ ```
+ """
+ ),
+ "error",
+ )
+
+ def test_report_single_file_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+ def test_report_multiple_files_multiple_testsuites(self):
+ self.assertEqual(
+ _generate_report(
+ "ABC and DEF",
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+
+
+
+ """
+ )
+ ),
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+
+
+
+
+
+ """
+ )
+ ),
+ ],
+ ),
+ self.MULTI_SUITE_OUTPUT,
+ )
+
+
+def _generate_report(title, junit_objects):
+ style = None
+
+ if not junit_objects:
+ return ("", style)
+
+ failures = {}
+ tests_run = 0
+ tests_skipped = 0
+ tests_failed = 0
+
+ for results in junit_objects:
+ for testsuite in results:
+ tests_run += testsuite.tests
+ tests_skipped += testsuite.skipped
+ tests_failed += testsuite.failures
+
+ for test in testsuite:
+ if (
+ not test.is_passed
+ and test.result
+ and isinstance(test.result[0], Failure)
+ ):
+ if failures.get(testsuite.name) is None:
+ failures[testsuite.name] = []
+ failures[testsuite.name].append(
+ (test.classname + "/" + test.name, test.result[0].text)
+ )
+
+ if not tests_run:
+ return ("", style)
+
+ style = "error" if tests_failed else "success"
+ report = [f"# {title}", ""]
+
+ tests_passed = tests_run - tests_skipped - tests_failed
+
+ def plural(num_tests):
+ return "test" if num_tests == 1 else "tests"
+
+ if tests_passed:
+ report.append(f"* {tests_passed} {plural(tests_passed)} passed")
+ if tests_skipped:
+ report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
+ if tests_failed:
+ report.append(f"* {tests_failed} {plural(tests_failed)} failed")
+
+ if failures:
+ report.extend(["", "## Failed tests", "(click to see output)"])
+ for testsuite_name, failures in failures.items():
+ report.extend(["", f"### {testsuite_name}"])
+ for name, output in failures:
+ report.extend(
+ [
+ "",
+ f"{name}
",
+ "",
+ "```",
+ output,
+ "```",
+ " ",
+ ]
+ )
+
+ return "\n".join(report), style
+
+
+def generate_report(title, junit_files):
+ return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "title", help="Title of the test report, without Markdown formatting."
+ )
+ parser.add_argument("context", help="Annotation context to write to.")
+ parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+ args = parser.parse_args()
+
+ report, style = generate_report(args.title, args.junit_files)
+ check_call(
+ [
+ "buildkite-agent",
+ "annotate",
+ "--context",
+ args.context,
+ "--style",
+ style,
+ report,
+ ]
+ )
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 17ea51c08faf..a4aeea7a16ad 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -28,11 +28,16 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
ccache --clear
fi
-function show-stats {
+function at-exit {
mkdir -p artifacts
ccache --print-stats > artifacts/ccache_stats.txt
+
+ # If building fails there will be no results files.
+ shopt -s nullglob
+ python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \
+ "linux-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
}
-trap show-stats EXIT
+trap at-exit EXIT
projects="${1}"
targets="${2}"
@@ -42,6 +47,7 @@ lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-outpu
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
+pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
-D LLVM_ENABLE_PROJECTS="${projects}" \
-G Ninja \
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 9ec44c22442d..4ead122212f4 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -27,17 +27,23 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
fi
sccache --zero-stats
-function show-stats {
+function at-exit {
mkdir -p artifacts
sccache --show-stats >> artifacts/sccache_stats.txt
+
+ # If building fails there will be no results files.
+ shopt -s nullglob
+ python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \
+ "windows-x64-test-results" "${BUILD_DIR}"/test-results.*.xml
}
-trap show-stats EXIT
+trap at-exit EXIT
projects="${1}"
targets="${2}"
echo "--- cmake"
pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
+pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
# The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
# on fixing a build reliability issue on the build server, please
diff --git a/.ci/requirements.txt b/.ci/requirements.txt
new file mode 100644
index 000000000000..ad63858c9fdc
--- /dev/null
+++ b/.ci/requirements.txt
@@ -0,0 +1 @@
+junitparser==3.2.0