llvm-project/.ci/generate-buildkite-pipeline-premerge

#!/usr/bin/env bash
#===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===----------------------------------------------------------------------===##

#
# This file generates a Buildkite pipeline that triggers the various CI jobs for
# the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
#
# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
#

reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
if [[ "${reviewID}" != "" ]]; then
  buildMessage="https://llvm.org/${reviewID}"
else
  buildMessage="Push to branch ${BUILDKITE_BRANCH}"
fi

cat <<EOF
steps:
EOF

# If libc++ or one of the runtimes directories changed
if git diff --name-only HEAD~1 | grep -q -E "^libcxx/|^libcxxabi/|^libunwind/|^runtimes/|^cmake/"; then
  cat <<EOF
  - trigger: "libcxx-ci"
    build:
      message: "${buildMessage}"
      commit: "${BUILDKITE_COMMIT}"
      branch: "${BUILDKITE_BRANCH}"
EOF
fi

# If clang changed
if git diff --name-only HEAD~1 | grep -q -E "^clang/"; then
  cat <<EOF
  - trigger: "clang-ci"
    build:
      message: "${buildMessage}"
      commit: "${BUILDKITE_COMMIT}"
      branch: "${BUILDKITE_BRANCH}"
EOF
fi

#
# If we're not running a more specific pipeline, generate a legacy monolithic pipeline
# based on which subdirectories have been modified. We have some heuristics for this
# to be reasonable.
#
# Individual projects should instead define the pre-commit CI tests that suits their
# needs while letting them run on the infrastructure provided by LLVM.
#
function compute-projects-to-test() {
  projects=${@}
  for project in ${projects}; do
    echo "${project}"
    case ${project} in
    lld)
      for p in bolt cross-project-tests; do
        echo $p
      done
    ;;
    llvm)
      for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do
        echo $p
      done
    ;;
    clang)
      for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do
        echo $p
      done
    ;;
    clang-tools-extra)
      echo libc
    ;;
    mlir)
      echo flang
    ;;
    *)
      # Nothing to do
    ;;
    esac
  done
}

function add-dependencies() {
  projects=${@}
  for project in ${projects}; do
    echo "${project}"
    case ${project} in
    bolt)
      for p in lld llvm; do
        echo $p
      done
    ;;
    cross-project-tests)
      for p in lld clang; do
        echo $p
      done
    ;;
    clang-tools-extra)
      for p in llvm clang; do
        echo $p
      done
    ;;
    compiler-rt|libc|openmp)
      echo clang
    ;;
    flang|lldb)
      for p in llvm clang; do
        echo $p
      done
    ;;
    lld|mlir|polly)
      echo llvm
    ;;
    *)
      # Nothing to do
    ;;
    esac
  done
}

function exclude-linux() {
  projects=${@}
  for project in ${projects}; do
    case ${project} in
    cross-project-tests) ;; # tests failing
    lldb)                ;; # tests failing
    openmp)              ;; # https://github.com/google/llvm-premerge-checks/issues/410
    *)
      echo "${project}"
    ;;
    esac
  done
}

function exclude-windows() {
  projects=${@}
  for project in ${projects}; do
    case ${project} in
    cross-project-tests) ;; # tests failing
    compiler-rt)         ;; # tests taking too long
    openmp)              ;; # TODO: having trouble with the Perl installation
    libc)                ;; # no Windows support
    lldb)                ;; # tests failing
    bolt)                ;; # tests are not supported yet
    *)
      echo "${project}"
    ;;
    esac
  done
}

function keep-modified-projects() {
  projects=${@}
  git_diff="$(git diff --name-only HEAD~1)"
  for project in ${projects}; do
    if echo "${git_diff}" | grep -q -E "^${project}/"; then
      echo "${project}"
    fi
  done
}

function check-targets() {
  projects=${@}
  for project in ${projects}; do
    case ${project} in
    clang-tools-extra)
      echo "check-clang-tools"
    ;;
    compiler-rt)
      echo "check-all"
    ;;
    cross-project-tests)
      echo "check-cross-project"
    ;;
    lldb)
      echo "check-all" # TODO: check-lldb may not include all the LLDB tests?
    ;;
    pstl)
      echo "check-all"
    ;;
    libclc)
      echo "check-all"
    ;;
    *)
      echo "check-${project}"
    ;;
    esac
  done
}

# Figure out which projects need to be built on each platform
all_projects="bolt clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
modified_projects="$(keep-modified-projects ${all_projects})"

linux_projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects}))
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)

windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)

# Generate the appropriate pipeline
if [[ "${linux_projects}" != "" ]]; then
  cat <<EOF
- label: ':linux: x64 Debian'
  artifact_paths:
    - '*_result.json'
    - 'build/monolithic-linux/test-results.xml'
  agents:
    queue: 'linux'
  retry:
    automatic:
      - exit_status: -1  # Agent was lost
        limit: 2
      - exit_status: 255
        limit: 2 # Forced agent shutdown
  timeout_in_minutes: 120
  env:
    CC: 'clang'
    CXX: 'clang++'
  commands:
    - './.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})"'
EOF
fi

if [[ "${windows_projects}" != "" ]]; then
  cat <<EOF
- label: ':windows: x64 Windows'
  artifact_paths:
    - '*_result.json'
    - 'build/monolithic-windows/test-results.xml'
  agents:
    queue: 'windows'
  retry:
    automatic:
      - exit_status: -1  # Agent was lost
        limit: 2
      - exit_status: 255
        limit: 2 # Forced agent shutdown
  timeout_in_minutes: 150
  env:
    CC: 'cl'
    CXX: 'cl'
    LD: 'link'
  commands:
    - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
    - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
EOF
fi