mirror of
https://github.com/llvm/llvm-project.git
synced 2025-04-14 17:06:38 +00:00
[CI] Move CI over to new project computation script
This patch migrates the CI over to the new compute_projects.py script for calculating what projects need to be tested based on a change to LLVM. Reviewers: lnihlen, ldionne, tstellar, Endilll, joker-eph, Keenuts Reviewed By: Keenuts, tstellar Pull Request: https://github.com/llvm/llvm-project/pull/132642
This commit is contained in:
Aiden Grossman
GitHub
parent
40c1d50024
commit
34d858635f
@ -1,194 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#===----------------------------------------------------------------------===##
|
||||
#
|
||||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
#===----------------------------------------------------------------------===##
|
||||
|
||||
#
|
||||
# This file contains functions to compute which projects should be built by CI
|
||||
# systems and is intended to provide common functionality applicable across
|
||||
# multiple systems during a transition period.
|
||||
#
|
||||
|
||||
function compute-projects-to-test() {
|
||||
isForWindows=$1
|
||||
shift
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
echo "${project}"
|
||||
case ${project} in
|
||||
lld)
|
||||
for p in bolt cross-project-tests; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
llvm)
|
||||
for p in bolt clang clang-tools-extra lld lldb mlir polly; do
|
||||
echo $p
|
||||
done
|
||||
# Flang is not stable in Windows CI at the moment
|
||||
if [[ $isForWindows == 0 ]]; then
|
||||
echo flang
|
||||
fi
|
||||
;;
|
||||
clang)
|
||||
# lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
|
||||
for p in clang-tools-extra compiler-rt cross-project-tests; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
clang-tools-extra)
|
||||
echo libc
|
||||
;;
|
||||
mlir)
|
||||
# Flang is not stable in Windows CI at the moment
|
||||
if [[ $isForWindows == 0 ]]; then
|
||||
echo flang
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Nothing to do
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function compute-runtimes-to-test() {
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
case ${project} in
|
||||
clang)
|
||||
for p in libcxx libcxxabi libunwind; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
*)
|
||||
# Nothing to do
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function add-dependencies() {
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
echo "${project}"
|
||||
case ${project} in
|
||||
bolt)
|
||||
for p in clang lld llvm; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
cross-project-tests)
|
||||
for p in lld clang; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
clang-tools-extra)
|
||||
for p in llvm clang; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
compiler-rt|libc|openmp)
|
||||
echo clang lld
|
||||
;;
|
||||
flang|lldb|libclc)
|
||||
for p in llvm clang; do
|
||||
echo $p
|
||||
done
|
||||
;;
|
||||
lld|mlir|polly)
|
||||
echo llvm
|
||||
;;
|
||||
*)
|
||||
# Nothing to do
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function exclude-linux() {
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
case ${project} in
|
||||
cross-project-tests) ;; # tests failing
|
||||
openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
|
||||
*)
|
||||
echo "${project}"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function exclude-windows() {
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
case ${project} in
|
||||
cross-project-tests) ;; # tests failing
|
||||
compiler-rt) ;; # tests taking too long
|
||||
openmp) ;; # TODO: having trouble with the Perl installation
|
||||
libc) ;; # no Windows support
|
||||
lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
|
||||
bolt) ;; # tests are not supported yet
|
||||
*)
|
||||
echo "${project}"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# Prints only projects that are both present in $modified_dirs and the passed
|
||||
# list.
|
||||
function keep-modified-projects() {
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
if echo "$modified_dirs" | grep -q -E "^${project}$"; then
|
||||
echo "${project}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function check-targets() {
|
||||
# Do not use "check-all" here because if there is "check-all" plus a
|
||||
# project specific target like "check-clang", that project's tests
|
||||
# will be run twice.
|
||||
projects=${@}
|
||||
for project in ${projects}; do
|
||||
case ${project} in
|
||||
clang-tools-extra)
|
||||
echo "check-clang-tools"
|
||||
;;
|
||||
compiler-rt)
|
||||
echo "check-compiler-rt"
|
||||
;;
|
||||
cross-project-tests)
|
||||
echo "check-cross-project"
|
||||
;;
|
||||
libcxx)
|
||||
echo "check-cxx"
|
||||
;;
|
||||
libcxxabi)
|
||||
echo "check-cxxabi"
|
||||
;;
|
||||
libunwind)
|
||||
echo "check-unwind"
|
||||
;;
|
||||
lldb)
|
||||
echo "check-lldb"
|
||||
;;
|
||||
pstl)
|
||||
# Currently we do not run pstl tests in CI.
|
||||
;;
|
||||
libclc)
|
||||
# Currently there is no testing for libclc.
|
||||
;;
|
||||
*)
|
||||
echo "check-${project}"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
|
||||
echo "Directories modified:" >&2
|
||||
echo "$modified_dirs" >&2
|
||||
|
||||
. ./.ci/compute-projects.sh
|
||||
|
||||
# Project specific pipelines.
|
||||
|
||||
# If libc++ or one of the runtimes directories changed.
|
||||
@ -73,20 +71,16 @@ fi
|
||||
# needs while letting them run on the infrastructure provided by LLVM.
|
||||
|
||||
# Figure out which projects need to be built on each platform
|
||||
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
|
||||
modified_projects="$(keep-modified-projects ${all_projects})"
|
||||
source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
|
||||
linux_projects=${projects_to_build}
|
||||
linux_check_targets=${project_check_targets}
|
||||
linux_runtimes=${runtimes_to_build}
|
||||
linux_runtime_check_targets=${runtimes_check_targets}
|
||||
|
||||
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
|
||||
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
|
||||
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
|
||||
source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
|
||||
windows_projects=${projects_to_build}
|
||||
windows_check_targets=${project_check_targets}
|
||||
|
||||
linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
|
||||
linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
|
||||
linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
|
||||
|
||||
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
|
||||
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
|
||||
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
|
||||
|
||||
# Generate the appropriate pipeline
|
||||
if [[ "${linux_projects}" != "" ]]; then
|
||||
|
||||
90
.github/workflows/premerge.yaml
vendored
90
.github/workflows/premerge.yaml
vendored
@ -49,39 +49,22 @@ jobs:
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
|
||||
modified_files=$(git diff --name-only HEAD~1...HEAD)
|
||||
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
|
||||
source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py)
|
||||
|
||||
echo $modified_files
|
||||
echo $modified_dirs
|
||||
|
||||
. ./.ci/compute-projects.sh
|
||||
|
||||
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
|
||||
modified_projects="$(keep-modified-projects ${all_projects})"
|
||||
|
||||
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
|
||||
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
|
||||
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
|
||||
|
||||
linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
|
||||
linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
|
||||
linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
|
||||
|
||||
if [[ "${linux_projects}" == "" ]]; then
|
||||
if [[ "${projects_to_build}" == "" ]]; then
|
||||
echo "No projects to build"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Building projects: ${linux_projects}"
|
||||
echo "Running project checks targets: ${linux_check_targets}"
|
||||
echo "Building runtimes: ${linux_runtimes}"
|
||||
echo "Running runtimes checks targets: ${linux_runtime_check_targets}"
|
||||
echo "Building projects: ${projects_to_build}"
|
||||
echo "Running project checks targets: ${project_check_targets}"
|
||||
echo "Building runtimes: ${runtimes_to_build}"
|
||||
echo "Running runtimes checks targets: ${runtimes_check_targets}"
|
||||
|
||||
export CC=/opt/llvm/bin/clang
|
||||
export CXX=/opt/llvm/bin/clang++
|
||||
|
||||
./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"
|
||||
./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
|
||||
|
||||
premerge-checks-windows:
|
||||
name: Windows Premerge Checks (Test Only - Please Ignore Results)
|
||||
@ -105,30 +88,17 @@ jobs:
|
||||
- name: Compute Projects
|
||||
id: vars
|
||||
run: |
|
||||
modified_files=$(git diff --name-only HEAD~1...HEAD)
|
||||
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
|
||||
source <(git diff --name-only HEAD~1...HEAD | python .ci/compute_projects.py)
|
||||
|
||||
echo $modified_files
|
||||
echo $modified_dirs
|
||||
|
||||
. ./.ci/compute-projects.sh
|
||||
|
||||
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
|
||||
modified_projects="$(keep-modified-projects ${all_projects})"
|
||||
|
||||
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
|
||||
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
|
||||
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
|
||||
|
||||
if [[ "${windows_projects}" == "" ]]; then
|
||||
if [[ "${projects_to_build}" == "" ]]; then
|
||||
echo "No projects to build"
|
||||
fi
|
||||
|
||||
echo "Building projects: ${windows_projects}"
|
||||
echo "Running project checks targets: ${windows_check_targets}"
|
||||
echo "Building projects: ${projects_to_build}"
|
||||
echo "Running project checks targets: ${project_check_targets}"
|
||||
|
||||
echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT
|
||||
echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT
|
||||
echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
|
||||
echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
|
||||
- name: Build and Test
|
||||
# Mark the job as a success even if the step fails so that people do
|
||||
# not get notified while the new premerge pipeline is in an
|
||||
@ -165,37 +135,15 @@ jobs:
|
||||
uses: llvm/actions/install-ninja@main
|
||||
- name: Build and Test
|
||||
run: |
|
||||
modified_files=$(git diff --name-only HEAD~1...HEAD)
|
||||
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
|
||||
source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
|
||||
|
||||
echo $modified_files
|
||||
echo $modified_dirs
|
||||
|
||||
. ./.ci/compute-projects.sh
|
||||
|
||||
all_projects="clang clang-tools-extra lld lldb llvm mlir"
|
||||
modified_projects="$(keep-modified-projects ${all_projects})"
|
||||
|
||||
# We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
|
||||
# and the lldb tests depend on libcxx, so we need to skip them.
|
||||
mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
|
||||
mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
|
||||
|
||||
mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
|
||||
mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
|
||||
mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
|
||||
|
||||
if [[ "${mac_projects}" == "" ]]; then
|
||||
if [[ "${projects_to_build}" == "" ]]; then
|
||||
echo "No projects to build"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Projects to test: ${modified_projects}"
|
||||
echo "Runtimes to test: ${mac_runtimes_to_test}"
|
||||
echo "Building projects: ${mac_projects}"
|
||||
echo "Running project checks targets: ${mac_check_targets}"
|
||||
echo "Building runtimes: ${mac_runtimes}"
|
||||
echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
|
||||
echo "Building projects: ${projects_to_build}"
|
||||
echo "Running project checks targets: ${project_check_targets}"
|
||||
|
||||
# -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
|
||||
# https://github.com/llvm/llvm-project/issues/81967
|
||||
@ -203,7 +151,7 @@ jobs:
|
||||
cmake -G Ninja \
|
||||
-B build \
|
||||
-S llvm \
|
||||
-DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \
|
||||
-DLLVM_ENABLE_PROJECTS="${projects_to_build}" \
|
||||
-DLLVM_DISABLE_ASSEMBLY_FILES=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLDB_INCLUDE_TESTS=OFF \
|
||||
@ -212,4 +160,4 @@ jobs:
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
|
||||
# The libcxx tests fail, so we are skipping the runtime targets.
|
||||
ninja -C build $mac_check_targets
|
||||
ninja -C build ${project_check_targets}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user