From 34d858635f0887cc083ccbe3ac69fd34f107b4f6 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 28 Mar 2025 22:25:52 -0700 Subject: [PATCH] [CI] Move CI over to new project computation script This patch migrates the CI over to the new compute_projects.py script for calculating what projects need to be tested based on a change to LLVM. Reviewers: lnihlen, ldionne, tstellar, Endilll, joker-eph, Keenuts Reviewed By: Keenuts, tstellar Pull Request: https://github.com/llvm/llvm-project/pull/132642 --- .ci/compute-projects.sh | 194 ----------------------- .ci/generate-buildkite-pipeline-premerge | 22 +-- .github/workflows/premerge.yaml | 90 +++-------- 3 files changed, 27 insertions(+), 279 deletions(-) delete mode 100644 .ci/compute-projects.sh diff --git a/.ci/compute-projects.sh b/.ci/compute-projects.sh deleted file mode 100644 index 32baf26b4f0a..000000000000 --- a/.ci/compute-projects.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -#===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -#===----------------------------------------------------------------------===## - -# -# This file contains functions to compute which projects should be built by CI -# systems and is intended to provide common functionality applicable across -# multiple systems during a transition period. -# - -function compute-projects-to-test() { - isForWindows=$1 - shift - projects=${@} - for project in ${projects}; do - echo "${project}" - case ${project} in - lld) - for p in bolt cross-project-tests; do - echo $p - done - ;; - llvm) - for p in bolt clang clang-tools-extra lld lldb mlir polly; do - echo $p - done - # Flang is not stable in Windows CI at the moment - if [[ $isForWindows == 0 ]]; then - echo flang - fi - ;; - clang) - # lldb is temporarily removed to alleviate Linux pre-commit CI waiting times - for p in clang-tools-extra compiler-rt cross-project-tests; do - echo $p - done - ;; - clang-tools-extra) - echo libc - ;; - mlir) - # Flang is not stable in Windows CI at the moment - if [[ $isForWindows == 0 ]]; then - echo flang - fi - ;; - *) - # Nothing to do - ;; - esac - done -} - -function compute-runtimes-to-test() { - projects=${@} - for project in ${projects}; do - case ${project} in - clang) - for p in libcxx libcxxabi libunwind; do - echo $p - done - ;; - *) - # Nothing to do - ;; - esac - done -} - -function add-dependencies() { - projects=${@} - for project in ${projects}; do - echo "${project}" - case ${project} in - bolt) - for p in clang lld llvm; do - echo $p - done - ;; - cross-project-tests) - for p in lld clang; do - echo $p - done - ;; - clang-tools-extra) - for p in llvm clang; do - echo $p - done - ;; - compiler-rt|libc|openmp) - echo clang lld - ;; - flang|lldb|libclc) - for p in llvm clang; do - echo $p - done - ;; - lld|mlir|polly) - echo llvm - ;; - *) - # Nothing to do - ;; - esac - done -} - -function exclude-linux() { - projects=${@} - for project in ${projects}; do - case ${project} in - cross-project-tests) ;; # tests failing - openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 - *) - echo "${project}" - ;; - esac - done -} - -function exclude-windows() { - projects=${@} - for project in ${projects}; do - case ${project} in - cross-project-tests) ;; # tests failing - compiler-rt) ;; # tests taking too long - openmp) ;; # TODO: having trouble with the Perl installation - libc) ;; # no Windows support - lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857) - bolt) ;; # tests are not supported yet - *) - echo "${project}" - ;; - esac - done -} - -# Prints only projects that are both present in $modified_dirs and the passed -# list. -function keep-modified-projects() { - projects=${@} - for project in ${projects}; do - if echo "$modified_dirs" | grep -q -E "^${project}$"; then - echo "${project}" - fi - done -} - -function check-targets() { - # Do not use "check-all" here because if there is "check-all" plus a - # project specific target like "check-clang", that project's tests - # will be run twice. - projects=${@} - for project in ${projects}; do - case ${project} in - clang-tools-extra) - echo "check-clang-tools" - ;; - compiler-rt) - echo "check-compiler-rt" - ;; - cross-project-tests) - echo "check-cross-project" - ;; - libcxx) - echo "check-cxx" - ;; - libcxxabi) - echo "check-cxxabi" - ;; - libunwind) - echo "check-unwind" - ;; - lldb) - echo "check-lldb" - ;; - pstl) - # Currently we do not run pstl tests in CI. - ;; - libclc) - # Currently there is no testing for libclc. - ;; - *) - echo "check-${project}" - ;; - esac - done -} - diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge index e547afaeb722..5e5f916f35b7 100755 --- a/.ci/generate-buildkite-pipeline-premerge +++ b/.ci/generate-buildkite-pipeline-premerge @@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u) echo "Directories modified:" >&2 echo "$modified_dirs" >&2 -. ./.ci/compute-projects.sh - # Project specific pipelines. # If libc++ or one of the runtimes directories changed. @@ -73,20 +71,16 @@ fi # needs while letting them run on the infrastructure provided by LLVM. # Figure out which projects need to be built on each platform -all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" -modified_projects="$(keep-modified-projects ${all_projects})" +source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux) +linux_projects=${projects_to_build} +linux_check_targets=${project_check_targets} +linux_runtimes=${runtimes_to_build} +linux_runtime_check_targets=${runtimes_check_targets} -linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects})) -linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq) -linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq) +source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows) +windows_projects=${projects_to_build} +windows_check_targets=${project_check_targets} -linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test}) -linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq) -linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq) - -windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects})) -windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq) -windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq) # Generate the appropriate pipeline if [[ "${linux_projects}" != "" ]]; then diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index c488421d3745..b7d2c7154762 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -49,39 +49,22 @@ jobs: run: | git config --global --add safe.directory '*' - modified_files=$(git diff --name-only HEAD~1...HEAD) - modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u) + source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py) - echo $modified_files - echo $modified_dirs - - . ./.ci/compute-projects.sh - - all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" - modified_projects="$(keep-modified-projects ${all_projects})" - - linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects})) - linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq) - linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq) - - linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test}) - linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq) - linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq) - - if [[ "${linux_projects}" == "" ]]; then + if [[ "${projects_to_build}" == "" ]]; then echo "No projects to build" exit 0 fi - echo "Building projects: ${linux_projects}" - echo "Running project checks targets: ${linux_check_targets}" - echo "Building runtimes: ${linux_runtimes}" - echo "Running runtimes checks targets: ${linux_runtime_check_targets}" + echo "Building projects: ${projects_to_build}" + echo "Running project checks targets: ${project_check_targets}" + echo "Building runtimes: ${runtimes_to_build}" + echo "Running runtimes checks targets: ${runtimes_check_targets}" export CC=/opt/llvm/bin/clang export CXX=/opt/llvm/bin/clang++ - ./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})" + ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" premerge-checks-windows: name: Windows Premerge Checks (Test Only - Please Ignore Results) @@ -105,30 +88,17 @@ jobs: - name: Compute Projects id: vars run: | - modified_files=$(git diff --name-only HEAD~1...HEAD) - modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq) + source <(git diff --name-only HEAD~1...HEAD | python .ci/compute_projects.py) - echo $modified_files - echo $modified_dirs - - . ./.ci/compute-projects.sh - - all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" - modified_projects="$(keep-modified-projects ${all_projects})" - - windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects})) - windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ') - windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';') - - if [[ "${windows_projects}" == "" ]]; then + if [[ "${projects_to_build}" == "" ]]; then echo "No projects to build" fi - echo "Building projects: ${windows_projects}" - echo "Running project checks targets: ${windows_check_targets}" + echo "Building projects: ${projects_to_build}" + echo "Running project checks targets: ${project_check_targets}" - echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT - echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT + echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT + echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT - name: Build and Test # Mark the job as a success even if the step fails so that people do # not get notified while the new premerge pipeline is in an @@ -165,37 +135,15 @@ jobs: uses: llvm/actions/install-ninja@main - name: Build and Test run: | - modified_files=$(git diff --name-only HEAD~1...HEAD) - modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u) + source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py) - echo $modified_files - echo $modified_dirs - - . ./.ci/compute-projects.sh - - all_projects="clang clang-tools-extra lld lldb llvm mlir" - modified_projects="$(keep-modified-projects ${all_projects})" - - # We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568 - # and the lldb tests depend on libcxx, so we need to skip them. - mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g') - mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ') - - mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects}) - mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ') - mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ') - - if [[ "${mac_projects}" == "" ]]; then + if [[ "${projects_to_build}" == "" ]]; then echo "No projects to build" exit 0 fi - echo "Projects to test: ${modified_projects}" - echo "Runtimes to test: ${mac_runtimes_to_test}" - echo "Building projects: ${mac_projects}" - echo "Running project checks targets: ${mac_check_targets}" - echo "Building runtimes: ${mac_runtimes}" - echo "Running runtimes checks targets: ${mac_runtime_check_targets}" + echo "Building projects: ${projects_to_build}" + echo "Running project checks targets: ${project_check_targets}" # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for # https://github.com/llvm/llvm-project/issues/81967 @@ -203,7 +151,7 @@ jobs: cmake -G Ninja \ -B build \ -S llvm \ - -DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \ + -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \ -DLLVM_DISABLE_ASSEMBLY_FILES=ON \ -DCMAKE_BUILD_TYPE=Release \ -DLLDB_INCLUDE_TESTS=OFF \ @@ -212,4 +160,4 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache # The libcxx tests fail, so we are skipping the runtime targets. - ninja -C build $mac_check_targets + ninja -C build ${project_check_targets}