[HIP] Support device sanitizer

Add option -fgpu-sanitize to enable sanitizer for AMDGPU target.

Since it is experimental, it is off by default.

Reviewed by: Artem Belevich

Differential Revision: https://reviews.llvm.org/D96835

clang/include/clang/Driver/Options.td

@@ -930,6 +930,9 @@ def gpu_max_threads_per_block_EQ : Joined<["--"], "gpu-max-threads-per-block=">,
 def gpu_instrument_lib_EQ : Joined<["--"], "gpu-instrument-lib=">,
   HelpText<"Instrument device library for HIP, which is a LLVM bitcode containing "
   "__cyg_profile_func_enter and __cyg_profile_func_exit">;
+defm gpu_sanitize : BoolFOption<"gpu-sanitize", EmptyKPM, DefaultFalse,
+  PosFlag<SetTrue, [], "Enable sanitizer for AMDGPU target">,
+  NegFlag<SetFalse>>;
 def cuid_EQ : Joined<["-"], "cuid=">, Flags<[CC1Option]>,
   HelpText<"An ID for compilation unit, which should be the same for the same "
            "compilation unit but different for different compilation units. "

clang/include/clang/Driver/ToolChain.h

@@ -663,6 +663,10 @@ public:
   virtual VersionTuple computeMSVCVersion(const Driver *D,
                                           const llvm::opt::ArgList &Args) const;
 
+  /// Get paths of HIP device libraries.
+  virtual llvm::SmallVector<std::string, 12>
+  getHIPDeviceLibs(const llvm::opt::ArgList &Args) const;
+
   /// Return sanitizers which are available in this toolchain.
   virtual SanitizerMask getSupportedSanitizers() const;
 

clang/lib/Driver/Driver.cpp

@@ -2844,12 +2844,15 @@ class OffloadingActionBuilder final {
   class HIPActionBuilder final : public CudaActionBuilderBase {
     /// The linker inputs obtained for each device arch.
     SmallVector<ActionList, 8> DeviceLinkerInputs;
+    bool GPUSanitize;
 
   public:
     HIPActionBuilder(Compilation &C, DerivedArgList &Args,
                      const Driver::InputList &Inputs)
         : CudaActionBuilderBase(C, Args, Inputs, Action::OFK_HIP) {
       DefaultCudaArch = CudaArch::GFX803;
+      GPUSanitize = Args.hasFlag(options::OPT_fgpu_sanitize,
+                                 options::OPT_fno_gpu_sanitize, false);
     }
 
     bool canUseBundlerUnbundler() const override { return true; }
@@ -2898,17 +2901,33 @@ class OffloadingActionBuilder final {
         // a fat binary containing all the code objects for different GPU's.
         // The fat binary is then an input to the host action.
         for (unsigned I = 0, E = GpuArchList.size(); I != E; ++I) {
+          if (GPUSanitize) {
+            // When GPU sanitizer is enabled, since we need to link in the
+            // the sanitizer runtime library after the sanitize pass, we have
+            // to skip the backend and assemble phases and use lld to link
+            // the bitcode.
+            ActionList AL;
+            AL.push_back(CudaDeviceActions[I]);
+            // Create a link action to link device IR with device library
+            // and generate ISA.
+            CudaDeviceActions[I] =
+                C.MakeAction<LinkJobAction>(AL, types::TY_Image);
+          } else {
+            // When GPU sanitizer is not enabled, we follow the conventional
+            // compiler phases, including backend and assemble phases.
+            ActionList AL;
             auto BackendAction = C.getDriver().ConstructPhaseAction(
                 C, Args, phases::Backend, CudaDeviceActions[I],
                 AssociatedOffloadKind);
             auto AssembleAction = C.getDriver().ConstructPhaseAction(
-              C, Args, phases::Assemble, BackendAction, AssociatedOffloadKind);
+                C, Args, phases::Assemble, BackendAction,
+                AssociatedOffloadKind);
+            AL.push_back(AssembleAction);
             // Create a link action to link device IR with device library
             // and generate ISA.
-          ActionList AL;
-          AL.push_back(AssembleAction);
             CudaDeviceActions[I] =
                 C.MakeAction<LinkJobAction>(AL, types::TY_Image);
+          }
 
           // OffloadingActionBuilder propagates device arch until an offload
           // action. Since the next action for creating fatbin does

clang/lib/Driver/SanitizerArgs.cpp

@@ -931,10 +931,15 @@ static bool hasTargetFeatureMTE(const llvm::opt::ArgStringList &CmdArgs) {
 void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
                             llvm::opt::ArgStringList &CmdArgs,
                             types::ID InputType) const {
-  // NVPTX/AMDGPU doesn't currently support sanitizers.  Bailing out here means
+  // NVPTX doesn't currently support sanitizers.  Bailing out here means
   // that e.g. -fsanitize=address applies only to host code, which is what we
   // want for now.
-  if (TC.getTriple().isNVPTX() || TC.getTriple().isAMDGPU())
+  //
+  // AMDGPU sanitizer support is experimental and controlled by -fgpu-sanitize.
+  if (TC.getTriple().isNVPTX() ||
+      (TC.getTriple().isAMDGPU() &&
+       !Args.hasFlag(options::OPT_fgpu_sanitize, options::OPT_fno_gpu_sanitize,
+                     false)))
     return;
 
   // Translate available CoverageFeatures to corresponding clang-cc1 flags.

clang/lib/Driver/ToolChain.cpp

@@ -1127,6 +1127,11 @@ void ToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
 void ToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                   ArgStringList &CC1Args) const {}
 
+llvm::SmallVector<std::string, 12>
+ToolChain::getHIPDeviceLibs(const ArgList &DriverArgs) const {
+  return {};
+}
+
 void ToolChain::AddIAMCUIncludeArgs(const ArgList &DriverArgs,
                                     ArgStringList &CC1Args) const {}
 

clang/lib/Driver/ToolChains/AMDGPU.cpp

@@ -50,6 +50,8 @@ void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
       OpenCL = FilePath;
     } else if (BaseName == "hip") {
       HIP = FilePath;
+    } else if (BaseName == "asanrtl") {
+      AsanRTL = FilePath;
     } else if (BaseName == "oclc_finite_only_off") {
       FiniteOnly.Off = FilePath;
     } else if (BaseName == "oclc_finite_only_on") {
@@ -605,47 +607,40 @@ void ROCMToolChain::addClangTargetOptions(
       DriverArgs.hasArg(options::OPT_cl_fp32_correctly_rounded_divide_sqrt);
 
   // Add the OpenCL specific bitcode library.
-  CC1Args.push_back("-mlink-builtin-bitcode");
+  llvm::SmallVector<std::string, 12> BCLibs;
-  CC1Args.push_back(DriverArgs.MakeArgString(RocmInstallation.getOpenCLPath()));
+  BCLibs.push_back(RocmInstallation.getOpenCLPath().str());
 
   // Add the generic set of libraries.
-  RocmInstallation.addCommonBitcodeLibCC1Args(
+  BCLibs.append(RocmInstallation.getCommonBitcodeLibs(
-      DriverArgs, CC1Args, LibDeviceFile, Wave64, DAZ, FiniteOnly,
+      DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
-      UnsafeMathOpt, FastRelaxedMath, CorrectSqrt);
+      FastRelaxedMath, CorrectSqrt));
+
+  llvm::for_each(BCLibs, [&](StringRef BCFile) {
+    CC1Args.push_back("-mlink-builtin-bitcode");
+    CC1Args.push_back(DriverArgs.MakeArgString(BCFile));
+  });
 }
 
-void RocmInstallationDetector::addCommonBitcodeLibCC1Args(
+llvm::SmallVector<std::string, 12>
-    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
+RocmInstallationDetector::getCommonBitcodeLibs(
-    StringRef LibDeviceFile, bool Wave64, bool DAZ, bool FiniteOnly,
+    const llvm::opt::ArgList &DriverArgs, StringRef LibDeviceFile, bool Wave64,
-    bool UnsafeMathOpt, bool FastRelaxedMath, bool CorrectSqrt) const {
+    bool DAZ, bool FiniteOnly, bool UnsafeMathOpt, bool FastRelaxedMath,
-  static const char LinkBitcodeFlag[] = "-mlink-builtin-bitcode";
+    bool CorrectSqrt) const {
 
-  CC1Args.push_back(LinkBitcodeFlag);
+  llvm::SmallVector<std::string, 12> BCLibs;
-  CC1Args.push_back(DriverArgs.MakeArgString(getOCMLPath()));
 
-  CC1Args.push_back(LinkBitcodeFlag);
+  auto AddBCLib = [&](StringRef BCFile) { BCLibs.push_back(BCFile.str()); };
-  CC1Args.push_back(DriverArgs.MakeArgString(getOCKLPath()));
 
-  CC1Args.push_back(LinkBitcodeFlag);
+  AddBCLib(getOCMLPath());
-  CC1Args.push_back(DriverArgs.MakeArgString(getDenormalsAreZeroPath(DAZ)));
+  AddBCLib(getOCKLPath());
+  AddBCLib(getDenormalsAreZeroPath(DAZ));
+  AddBCLib(getUnsafeMathPath(UnsafeMathOpt || FastRelaxedMath));
+  AddBCLib(getFiniteOnlyPath(FiniteOnly || FastRelaxedMath));
+  AddBCLib(getCorrectlyRoundedSqrtPath(CorrectSqrt));
+  AddBCLib(getWavefrontSize64Path(Wave64));
+  AddBCLib(LibDeviceFile);
 
-  CC1Args.push_back(LinkBitcodeFlag);
+  return BCLibs;
-  CC1Args.push_back(DriverArgs.MakeArgString(
-      getUnsafeMathPath(UnsafeMathOpt || FastRelaxedMath)));
-
-  CC1Args.push_back(LinkBitcodeFlag);
-  CC1Args.push_back(DriverArgs.MakeArgString(
-      getFiniteOnlyPath(FiniteOnly || FastRelaxedMath)));
-
-  CC1Args.push_back(LinkBitcodeFlag);
-  CC1Args.push_back(
-      DriverArgs.MakeArgString(getCorrectlyRoundedSqrtPath(CorrectSqrt)));
-
-  CC1Args.push_back(LinkBitcodeFlag);
-  CC1Args.push_back(DriverArgs.MakeArgString(getWavefrontSize64Path(Wave64)));
-
-  CC1Args.push_back(LinkBitcodeFlag);
-  CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile));
 }
 
 bool AMDGPUToolChain::shouldSkipArgument(const llvm::opt::Arg *A) const {

clang/lib/Driver/ToolChains/HIP.cpp

@@ -35,23 +35,6 @@ using namespace llvm::opt;
 
 namespace {
 const unsigned HIPCodeObjectAlign = 4096;
-
-static void addBCLib(const Driver &D, const ArgList &Args,
-                     ArgStringList &CmdArgs, ArgStringList LibraryPaths,
-                     StringRef BCName) {
-  StringRef FullName;
-  for (std::string LibraryPath : LibraryPaths) {
-    SmallString<128> Path(LibraryPath);
-    llvm::sys::path::append(Path, BCName);
-    FullName = Path;
-    if (llvm::sys::fs::exists(FullName)) {
-      CmdArgs.push_back("-mlink-builtin-bitcode");
-      CmdArgs.push_back(Args.MakeArgString(FullName));
-      return;
-    }
-  }
-  D.Diag(diag::err_drv_no_such_file) << BCName;
-}
 } // namespace
 
 void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
@@ -96,6 +79,13 @@ void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
   LldArgs.append({"-o", Output.getFilename()});
   for (auto Input : Inputs)
     LldArgs.push_back(Input.getFilename());
+
+  if (Args.hasFlag(options::OPT_fgpu_sanitize, options::OPT_fno_gpu_sanitize,
+                   false))
+    llvm::for_each(TC.getHIPDeviceLibs(Args), [&](StringRef BCFile) {
+      LldArgs.push_back(Args.MakeArgString(BCFile));
+    });
+
   const char *Lld = Args.MakeArgString(getToolChain().GetProgramPath("lld"));
   C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
                                          Lld, LldArgs, Inputs, Output));
@@ -247,13 +237,8 @@ void HIPToolChain::addClangTargetOptions(
     Action::OffloadKind DeviceOffloadingKind) const {
   HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
 
-  StringRef GpuArch = getGPUArch(DriverArgs);
-  assert(!GpuArch.empty() && "Must have an explicit GPU arch.");
-  (void) GpuArch;
   assert(DeviceOffloadingKind == Action::OFK_HIP &&
          "Only HIP offloading kinds are supported for GPUs.");
-  auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
-  const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind);
 
   CC1Args.push_back("-fcuda-is-device");
 
@@ -283,66 +268,10 @@ void HIPToolChain::addClangTargetOptions(
     CC1Args.push_back("-fapply-global-visibility-to-externs");
   }
 
-  if (DriverArgs.hasArg(options::OPT_nogpulib))
+  llvm::for_each(getHIPDeviceLibs(DriverArgs), [&](StringRef BCFile) {
-    return;
-  ArgStringList LibraryPaths;
-
-  // Find in --hip-device-lib-path and HIP_LIBRARY_PATH.
-  for (auto Path : RocmInstallation.getRocmDeviceLibPathArg())
-    LibraryPaths.push_back(DriverArgs.MakeArgString(Path));
-
-  addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
-
-  // Maintain compatability with --hip-device-lib.
-  auto BCLibs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
-  if (!BCLibs.empty()) {
-    for (auto Lib : BCLibs)
-      addBCLib(getDriver(), DriverArgs, CC1Args, LibraryPaths, Lib);
-  } else {
-    if (!RocmInstallation.hasDeviceLibrary()) {
-      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
-      return;
-    }
-
-    std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
-    if (LibDeviceFile.empty()) {
-      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GpuArch;
-      return;
-    }
-
-    // If --hip-device-lib is not set, add the default bitcode libraries.
-    // TODO: There are way too many flags that change this. Do we need to check
-    // them all?
-    bool DAZ = DriverArgs.hasFlag(options::OPT_fcuda_flush_denormals_to_zero,
-                                  options::OPT_fno_cuda_flush_denormals_to_zero,
-                                  getDefaultDenormsAreZeroForTarget(Kind));
-    // TODO: Check standard C++ flags?
-    bool FiniteOnly = false;
-    bool UnsafeMathOpt = false;
-    bool FastRelaxedMath = false;
-    bool CorrectSqrt = true;
-    bool Wave64 = isWave64(DriverArgs, Kind);
-
-    // Add the HIP specific bitcode library.
     CC1Args.push_back("-mlink-builtin-bitcode");
-    CC1Args.push_back(DriverArgs.MakeArgString(RocmInstallation.getHIPPath()));
+    CC1Args.push_back(DriverArgs.MakeArgString(BCFile));
-
+  });
-    // Add the generic set of libraries.
-    RocmInstallation.addCommonBitcodeLibCC1Args(
-      DriverArgs, CC1Args, LibDeviceFile, Wave64, DAZ, FiniteOnly,
-      UnsafeMathOpt, FastRelaxedMath, CorrectSqrt);
-
-    // Add instrument lib.
-    auto InstLib =
-        DriverArgs.getLastArgValue(options::OPT_gpu_instrument_lib_EQ);
-    if (InstLib.empty())
-      return;
-    if (llvm::sys::fs::exists(InstLib)) {
-      CC1Args.push_back("-mlink-builtin-bitcode");
-      CC1Args.push_back(DriverArgs.MakeArgString(InstLib));
-    } else
-      getDriver().Diag(diag::err_drv_no_such_file) << InstLib;
-  }
 }
 
 llvm::opt::DerivedArgList *
@@ -421,3 +350,99 @@ VersionTuple HIPToolChain::computeMSVCVersion(const Driver *D,
                                                const ArgList &Args) const {
   return HostTC.computeMSVCVersion(D, Args);
 }
+
+llvm::SmallVector<std::string, 12>
+HIPToolChain::getHIPDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
+  llvm::SmallVector<std::string, 12> BCLibs;
+  if (DriverArgs.hasArg(options::OPT_nogpulib))
+    return {};
+  ArgStringList LibraryPaths;
+
+  // Find in --hip-device-lib-path and HIP_LIBRARY_PATH.
+  for (auto Path : RocmInstallation.getRocmDeviceLibPathArg())
+    LibraryPaths.push_back(DriverArgs.MakeArgString(Path));
+
+  addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
+
+  // Maintain compatability with --hip-device-lib.
+  auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
+  if (!BCLibArgs.empty()) {
+    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+      StringRef FullName;
+      for (std::string LibraryPath : LibraryPaths) {
+        SmallString<128> Path(LibraryPath);
+        llvm::sys::path::append(Path, BCName);
+        FullName = Path;
+        if (llvm::sys::fs::exists(FullName)) {
+          BCLibs.push_back(FullName.str());
+          return;
+        }
+      }
+      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    });
+  } else {
+    if (!RocmInstallation.hasDeviceLibrary()) {
+      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
+      return {};
+    }
+    StringRef GpuArch = getGPUArch(DriverArgs);
+    assert(!GpuArch.empty() && "Must have an explicit GPU arch.");
+    (void)GpuArch;
+    auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
+    const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind);
+
+    std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
+    if (LibDeviceFile.empty()) {
+      getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GpuArch;
+      return {};
+    }
+
+    // If --hip-device-lib is not set, add the default bitcode libraries.
+    // TODO: There are way too many flags that change this. Do we need to check
+    // them all?
+    bool DAZ = DriverArgs.hasFlag(options::OPT_fcuda_flush_denormals_to_zero,
+                                  options::OPT_fno_cuda_flush_denormals_to_zero,
+                                  getDefaultDenormsAreZeroForTarget(Kind));
+    // TODO: Check standard C++ flags?
+    bool FiniteOnly = false;
+    bool UnsafeMathOpt = false;
+    bool FastRelaxedMath = false;
+    bool CorrectSqrt = true;
+    bool Wave64 = isWave64(DriverArgs, Kind);
+
+    if (DriverArgs.hasFlag(options::OPT_fgpu_sanitize,
+                           options::OPT_fno_gpu_sanitize, false)) {
+      auto AsanRTL = RocmInstallation.getAsanRTLPath();
+      if (AsanRTL.empty()) {
+        unsigned DiagID = getDriver().getDiags().getCustomDiagID(
+            DiagnosticsEngine::Error,
+            "AMDGPU address sanitizer runtime library (asanrtl) is not found. "
+            "Please install ROCm device library which supports address "
+            "sanitizer");
+        getDriver().Diag(DiagID);
+        return {};
+      } else
+        BCLibs.push_back(AsanRTL.str());
+    }
+
+    // Add the HIP specific bitcode library.
+    BCLibs.push_back(RocmInstallation.getHIPPath().str());
+
+    // Add the generic set of libraries.
+    BCLibs.append(RocmInstallation.getCommonBitcodeLibs(
+        DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
+        FastRelaxedMath, CorrectSqrt));
+
+    // Add instrument lib.
+    auto InstLib =
+        DriverArgs.getLastArgValue(options::OPT_gpu_instrument_lib_EQ);
+    if (InstLib.empty())
+      return BCLibs;
+    if (llvm::sys::fs::exists(InstLib))
+      BCLibs.push_back(InstLib.str());
+    else
+      getDriver().Diag(diag::err_drv_no_such_file) << InstLib;
+  }
+
+  return BCLibs;
+}

clang/lib/Driver/ToolChains/HIP.h

@@ -83,6 +83,8 @@ public:
                            llvm::opt::ArgStringList &CC1Args) const override;
   void AddHIPIncludeArgs(const llvm::opt::ArgList &DriverArgs,
                          llvm::opt::ArgStringList &CC1Args) const override;
+  llvm::SmallVector<std::string, 12>
+  getHIPDeviceLibs(const llvm::opt::ArgList &Args) const override;
 
   SanitizerMask getSupportedSanitizers() const override;
 

clang/lib/Driver/ToolChains/ROCm.h

@@ -88,6 +88,9 @@ private:
   SmallString<0> OpenCL;
   SmallString<0> HIP;
 
+  // Asan runtime library
+  SmallString<0> AsanRTL;
+
   // Libraries swapped based on compile flags.
   ConditionalLibrary WavefrontSize64;
   ConditionalLibrary FiniteOnly;
@@ -112,11 +115,12 @@ public:
                            bool DetectHIPRuntime = true,
                            bool DetectDeviceLib = false);
 
-  /// Add arguments needed to link default bitcode libraries.
+  /// Get file paths of default bitcode libraries common to AMDGPU based
-  void addCommonBitcodeLibCC1Args(const llvm::opt::ArgList &DriverArgs,
+  /// toolchains.
-                                  llvm::opt::ArgStringList &CC1Args,
+  llvm::SmallVector<std::string, 12>
-                                  StringRef LibDeviceFile, bool Wave64,
+  getCommonBitcodeLibs(const llvm::opt::ArgList &DriverArgs,
-                                  bool DAZ, bool FiniteOnly, bool UnsafeMathOpt,
+                       StringRef LibDeviceFile, bool Wave64, bool DAZ,
+                       bool FiniteOnly, bool UnsafeMathOpt,
                        bool FastRelaxedMath, bool CorrectSqrt) const;
 
   /// Check whether we detected a valid HIP runtime.
@@ -166,6 +170,9 @@ public:
     return HIP;
   }
 
+  /// Returns empty string of Asan runtime library is not available.
+  StringRef getAsanRTLPath() const { return AsanRTL; }
+
   StringRef getWavefrontSize64Path(bool Enabled) const {
     return WavefrontSize64.get(Enabled);
   }

clang/test/Driver/Inputs/rocm-invalid/README

@@ -0,0 +1,4 @@
+This directory is for testing invalid ROCm installations.
+
+It is invalid for -fgpu-sanitize since the required Asan
+runtime library (asanrtl.bc) is missing.

clang/test/Driver/Inputs/rocm-invalid/bin/.hipVersion

@@ -0,0 +1,6 @@
+# Auto-generated by cmake
+# NOTE: The trailing whitespace is added on purpose to verify that these
+# whitespaces are trimmed before paring.
+HIP_VERSION_MAJOR=3
+HIP_VERSION_MINOR=6
+HIP_VERSION_PATCH=20214-a2917cd

clang/test/Driver/hip-sanitize-options.hip

@@ -1,9 +1,40 @@
 // REQUIRES: clang-driver, x86-registered-target, amdgpu-registered-target
 
-// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx906 \
+// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx900 \
 // RUN:   -fsanitize=address \
-// RUN:   -nogpuinc -nogpulib \
+// RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck %s
 
+// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx900 \
+// RUN:   -fsanitize=address -fno-gpu-sanitize \
+// RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
+// RUN:   %s 2>&1 | FileCheck %s
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx900 \
+// RUN:   -fsanitize=address -fgpu-sanitize \
+// RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=NORDC %s
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx900 \
+// RUN:   -fsanitize=address -fgpu-sanitize -fgpu-rdc \
+// RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=RDC %s
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu --offload-arch=gfx900 \
+// RUN:   -fsanitize=address -fgpu-sanitize \
+// RUN:   -nogpuinc --rocm-path=%S/Inputs/rocm-invalid \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=FAIL %s
+
 // CHECK-NOT: {{"[^"]*clang[^"]*".* "-fcuda-is-device".* "-fsanitize=address"}}
+// CHECK-NOT: {{"[^"]*lld[^"]*".* ".*hip.bc"}}
 // CHECK: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
+
+// NORDC: {{"[^"]*clang[^"]*".* "-fcuda-is-device".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.bc]]"
+// NORDC: {{"[^"]*lld[^"]*".*}} "[[OUT]]" {{".*asanrtl.bc" ".*hip.bc"}}
+// NORDC: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
+
+// RDC: {{"[^"]*clang[^"]*".* "-triple" "x86_64-unknown-linux-gnu".* "-fsanitize=address"}}
+// RDC: {{"[^"]*clang[^"]*".* "-emit-llvm-bc".* "-fcuda-is-device".* "-fsanitize=address".*}} "-o" "[[OUT:[^"]*.bc]]"
+// RDC: {{"[^"]*lld[^"]*".*}} "[[OUT]]" {{".*asanrtl.bc" ".*hip.bc"}}
+
+// FAIL: AMDGPU address sanitizer runtime library (asanrtl) is not found. Please install ROCm device library which supports address sanitizer