[asancov] Write coverage directly to a memory-mapped file.

This way does not require a __sanitizer_cov_dump() call. That's
important on Android, where apps can be killed at arbitrary time.

We write raw PCs to disk instead of module offsets; we also write
memory layout to a separate file. This increases dump size by the
factor of 2 on 64-bit systems.

llvm-svn: 209653

compiler-rt/lib/asan/asan_interceptors.cc

@@ -146,6 +146,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
   } while (false)
 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) CovUpdateMapping()
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() CovUpdateMapping()
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 
 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) ASAN_READ_RANGE(p, s)

compiler-rt/lib/msan/msan_interceptors.cc

@@ -886,25 +886,6 @@ INTERCEPTOR(char *, dlerror, int fake) {
   return res;
 }
 
-// dlopen() ultimately calls mmap() down inside the loader, which generally
-// doesn't participate in dynamic symbol resolution.  Therefore we won't
-// intercept its calls to mmap, and we have to hook it here.  The loader
-// initializes the module before returning, so without the dynamic component, we
-// won't be able to clear the shadow before the initializers.  Fixing this would
-// require putting our own initializer first to clear the shadow.
-INTERCEPTOR(void *, dlopen, const char *filename, int flag) {
-  ENSURE_MSAN_INITED();
-  EnterLoader();
-  link_map *map = (link_map *)REAL(dlopen)(filename, flag);
-  ExitLoader();
-  if (!__msan_has_dynamic_component() && map) {
-    // If msandr didn't clear the shadow before the initializers ran, we do it
-    // ourselves afterwards.
-    ForEachMappedRegion(map, __msan_unpoison);
-  }
-  return (void *)map;
-}
-
 typedef int (*dl_iterate_phdr_cb)(__sanitizer_dl_phdr_info *info, SIZE_T size,
                                   void *data);
 struct dl_iterate_phdr_data {
@@ -1234,6 +1215,13 @@ int OnExit() {
   } while (false)  // FIXME
 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, map)                       \
+  if (!__msan_has_dynamic_component() && map) {                                \
+    /* If msandr didn't clear the shadow before the initializers ran, we do */ \
+    /* it ourselves afterwards. */                                             \
+    ForEachMappedRegion((link_map *)map, __msan_unpoison);                     \
+  }
+
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 
 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
@@ -1559,7 +1547,6 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(recvfrom);
   INTERCEPT_FUNCTION(dladdr);
   INTERCEPT_FUNCTION(dlerror);
-  INTERCEPT_FUNCTION(dlopen);
   INTERCEPT_FUNCTION(dl_iterate_phdr);
   INTERCEPT_FUNCTION(getrusage);
   INTERCEPT_FUNCTION(sigaction);

compiler-rt/lib/sanitizer_common/CMakeLists.txt

@@ -4,7 +4,6 @@
 set(SANITIZER_SOURCES
   sanitizer_allocator.cc
   sanitizer_common.cc
-  sanitizer_coverage.cc
   sanitizer_deadlock_detector1.cc
   sanitizer_deadlock_detector2.cc
   sanitizer_flags.cc
@@ -31,6 +30,8 @@ set(SANITIZER_SOURCES
 
 set(SANITIZER_LIBCDEP_SOURCES
   sanitizer_common_libcdep.cc
+  sanitizer_coverage_libcdep.cc
+  sanitizer_coverage_mapping_libcdep.cc
   sanitizer_linux_libcdep.cc
   sanitizer_posix_libcdep.cc
   sanitizer_stacktrace_libcdep.cc

compiler-rt/lib/sanitizer_common/sanitizer_common.cc

@@ -263,11 +263,6 @@ void DecreaseTotalMmap(uptr size) {
   atomic_fetch_sub(&g_total_mmaped, size, memory_order_relaxed);
 }
 
-static void (*sandboxing_callback)();
-void SetSandboxingCallback(void (*f)()) {
-  sandboxing_callback = f;
-}
-
 }  // namespace __sanitizer
 
 using namespace __sanitizer;  // NOLINT
@@ -300,13 +295,6 @@ void __sanitizer_set_report_path(const char *path) {
   }
 }
 
-void NOINLINE
-__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args) {
-  PrepareForSandboxing(args);
-  if (sandboxing_callback)
-    sandboxing_callback();
-}
-
 void __sanitizer_report_error_summary(const char *error_summary) {
   Printf("%s\n", error_summary);
 }

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -161,6 +161,7 @@ uptr ReadFileToBuffer(const char *file_name, char **buff,
 // (or NULL if the mapping failes). Stores the size of mmaped region
 // in '*buff_size'.
 void *MapFileToMemory(const char *file_name, uptr *buff_size);
+void *MapWritableFileToMemory(void *addr, uptr size, uptr fd, uptr offset);
 
 // Error report formatting.
 const char *StripPathPrefix(const char *filepath,
@@ -187,6 +188,8 @@ void PrepareForSandboxing(__sanitizer_sandbox_arguments *args);
 void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
 void SetSandboxingCallback(void (*f)());
 
+void CovUpdateMapping();
+
 void InitTlsSize();
 uptr GetTlsSize();
 
@@ -479,6 +482,10 @@ class LoadedModule {
   const char *full_name() const { return full_name_; }
   uptr base_address() const { return base_address_; }
 
+  uptr n_ranges() const { return n_ranges_; }
+  uptr address_range_start(int i) const { return ranges_[i].beg; }
+  uptr address_range_end(int i) const { return ranges_[i].end; }
+
  private:
   struct AddressRange {
     uptr beg;

compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc

@@ -75,6 +75,19 @@
 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) {}
 #endif
 
+#ifndef COMMON_INTERCEPTOR_LIBRARY_LOADED
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, map) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_LIBRARY_UNLOADED
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_ENTER_NOIGNORE
+#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, ...) \
+  COMMON_INTERCEPTOR_ENTER(ctx, __VA_ARGS__)
+#endif
+
 struct FileMetadata {
   // For open_memstream().
   char **addr;
@@ -4036,6 +4049,29 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {
 #define INIT_FCLOSE
 #endif
 
+#if SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
+INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
+  void *res = REAL(dlopen)(filename, flag);
+  COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
+  return res;
+}
+
+INTERCEPTOR(int, dlclose, void *handle) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle);
+  int res = REAL(dlclose)(handle);
+  COMMON_INTERCEPTOR_LIBRARY_UNLOADED();
+  return res;
+}
+#define INIT_DLOPEN_DLCLOSE          \
+  COMMON_INTERCEPT_FUNCTION(dlopen); \
+  COMMON_INTERCEPT_FUNCTION(dlclose);
+#else
+#define INIT_DLOPEN_DLCLOSE
+#endif
+
 static void InitializeCommonInterceptors() {
   static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
   interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap();
@@ -4177,4 +4213,5 @@ static void InitializeCommonInterceptors() {
   INIT_OBSTACK;
   INIT_FFLUSH;
   INIT_FCLOSE;
+  INIT_DLOPEN_DLCLOSE;
 }

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc

@@ -41,4 +41,17 @@ bool ColorizeReports() {
   return internal_strcmp(flag, "always") == 0 ||
          (internal_strcmp(flag, "auto") == 0 && PrintsToTtyCached());
 }
+
+static void (*sandboxing_callback)();
+void SetSandboxingCallback(void (*f)()) {
+  sandboxing_callback = f;
+}
+
 }  // namespace __sanitizer
+
+void NOINLINE
+__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args) {
+  PrepareForSandboxing(args);
+  if (sandboxing_callback)
+    sandboxing_callback();
+}

compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc

@@ -45,11 +45,10 @@ atomic_uint32_t dump_once_guard;  // Ensure that CovDump runs only once.
 // pc_array is the array containing the covered PCs.
 // To make the pc_array thread- and async-signal-safe it has to be large enough.
 // 128M counters "ought to be enough for anybody" (4M on 32-bit).
-// pc_array is allocated with MmapNoReserveOrDie and so it uses only as
-// much RAM as it really needs.
-static const uptr kPcArraySize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
-static uptr *pc_array;
-static atomic_uintptr_t pc_array_index;
+
+// With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file.
+// In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping()
+// dump current memory layout to another file.
 
 static bool cov_sandboxed = false;
 static int cov_fd = kInvalidFd;
@@ -57,22 +56,114 @@ static unsigned int cov_max_block_size = 0;
 
 namespace __sanitizer {
 
+class CoverageData {
+ public:
+  void Init();
+  void Extend(uptr npcs);
+  void Add(uptr pc);
+
+  uptr *data();
+  uptr size();
+
+ private:
+  // Maximal size pc array may ever grow.
+  // We MmapNoReserve this space to ensure that the array is contiguous.
+  static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
+  // The amount file mapping for the pc array is grown by.
+  static const uptr kPcArrayMmapSize = 64 * 1024;
+
+  // pc_array is allocated with MmapNoReserveOrDie and so it uses only as
+  // much RAM as it really needs.
+  uptr *pc_array;
+  // Index of the first available pc_array slot.
+  atomic_uintptr_t pc_array_index;
+  // Array size.
+  atomic_uintptr_t pc_array_size;
+  // Current file mapped size of the pc array.
+  uptr pc_array_mapped_size;
+  // Descriptor of the file mapped pc array.
+  int pc_fd;
+  StaticSpinMutex mu;
+
+  void DirectInit();
+};
+
+static CoverageData coverage_data;
+
+void CoverageData::DirectInit() {
+  InternalScopedString path(64);
+  internal_snprintf((char *)path.data(), path.size(), "%zd.sancov.raw",
+                    internal_getpid());
+  pc_fd = OpenFile(path.data(), true);
+  if (internal_iserror(pc_fd)) {
+    Report(" Coverage: failed to open %s for writing\n", path.data());
+    Die();
+  }
+
+  atomic_store(&pc_array_size, 0, memory_order_relaxed);
+  pc_array_mapped_size = 0;
+
+  CovUpdateMapping();
+}
+
+void CoverageData::Init() {
+  pc_array = reinterpret_cast<uptr *>(
+      MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
+  if (common_flags()->coverage_direct) {
+    DirectInit();
+  } else {
+    pc_fd = 0;
+    atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
+  }
+}
+
+// Extend coverage PC array to fit additional npcs elements.
+void CoverageData::Extend(uptr npcs) {
+  // If pc_fd=0, pc array is a huge anonymous mapping that does not need to be
+  // resized.
+  if (!pc_fd) return;
+  SpinMutexLock l(&mu);
+
+  uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
+  size += npcs * sizeof(uptr);
+
+  if (size > pc_array_mapped_size) {
+    uptr new_mapped_size = pc_array_mapped_size;
+    while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize;
+
+    // Extend the file and map the new space at the end of pc_array.
+    uptr res = internal_ftruncate(pc_fd, new_mapped_size);
+    int err;
+    if (internal_iserror(res, &err)) {
+      Printf("failed to extend raw coverage file: %d\n", err);
+      Die();
+    }
+    void *p = MapWritableFileToMemory(pc_array + pc_array_mapped_size,
+                                      new_mapped_size - pc_array_mapped_size,
+                                      pc_fd, pc_array_mapped_size);
+    CHECK_EQ(p, pc_array + pc_array_mapped_size);
+    pc_array_mapped_size = new_mapped_size;
+  }
+
+  atomic_store(&pc_array_size, size, memory_order_release);
+}
+
 // Simply add the pc into the vector under lock. If the function is called more
 // than once for a given PC it will be inserted multiple times, which is fine.
-static void CovAdd(uptr pc) {
+void CoverageData::Add(uptr pc) {
   if (!pc_array) return;
   uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
-  CHECK_LT(idx, kPcArraySize);
+  CHECK_LT(idx * sizeof(uptr),
+           atomic_load(&pc_array_size, memory_order_acquire));
   pc_array[idx] = pc;
 }
 
-void CovInit() {
-  pc_array = reinterpret_cast<uptr *>(
-      MmapNoReserveOrDie(sizeof(uptr) * kPcArraySize, "CovInit"));
+uptr *CoverageData::data() {
+  return pc_array;
 }
 
-static inline bool CompareLess(const uptr &a, const uptr &b) {
-  return a < b;
+uptr CoverageData::size() {
+  return atomic_load(&pc_array_index, memory_order_relaxed);
 }
 
 // Block layout for packed file format: header, followed by module name (no
@@ -150,15 +241,15 @@ static int CovOpenFile(bool packed, const char* name) {
 
 // Dump the coverage on disk.
 static void CovDump() {
-  if (!common_flags()->coverage) return;
+  if (!common_flags()->coverage || common_flags()->coverage_direct) return;
 #if !SANITIZER_WINDOWS
   if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
     return;
-  uptr size = atomic_load(&pc_array_index, memory_order_relaxed);
-  InternalSort(&pc_array, size, CompareLess);
+  uptr size = coverage_data.size();
   InternalMmapVector<u32> offsets(size);
-  const uptr *vb = pc_array;
-  const uptr *ve = vb + size;
+  uptr *vb = coverage_data.data();
+  uptr *ve = vb + size;
+  SortArray(vb, size);
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
   uptr mb, me, off, prot;
   InternalScopedBuffer<char> module(4096);
@@ -227,10 +318,15 @@ int MaybeOpenCovFile(const char *name) {
 
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov() {
-  CovAdd(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()));
+  coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()));
 }
 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); }
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { CovInit(); }
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
+  coverage_data.Init();
+}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) {
+  coverage_data.Extend(npcs);
+}
 SANITIZER_INTERFACE_ATTRIBUTE
 sptr __sanitizer_maybe_open_cov_file(const char *name) {
   return MaybeOpenCovFile(name);

compiler-rt/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc

@@ -0,0 +1,97 @@
+//===-- sanitizer_coverage_mapping.cc -------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Mmap-based implementation of sanitizer coverage.
+//
+// This is part of the implementation of code coverage that does not require
+// __sanitizer_cov_dump() call. Data is stored in 2 files per process.
+//
+// $pid.sancov.map describes process memory layout in the following text-based
+// format:
+// <pointer size in bits>  // 1 line, 32 or 64
+// <mapping start> <mapping end> <base address> <dso name> // repeated
+// ...
+// Mapping lines are NOT sorted. This file is updated every time memory layout
+// is changed (i.e. in dlopen() and dlclose() interceptors).
+//
+// $pid.sancov.raw is a binary dump of PC values, sizeof(uptr) each. Again, not
+// sorted. This file is extended by 64Kb at a time and mapped into memory. It
+// contains one or more 0 words at the end, up to the next 64Kb aligned offset.
+//
+// To convert these 2 files to the usual .sancov format, run sancov.py rawunpack
+// $pid.sancov.raw.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_procmaps.h"
+
+namespace __sanitizer {
+
+static const uptr kMaxNumberOfModules = 1 << 14;
+
+void CovUpdateMapping() {
+  if (!common_flags()->coverage || !common_flags()->coverage_direct) return;
+
+  int err;
+  InternalScopedString tmp_path(64);
+  internal_snprintf((char *)tmp_path.data(), tmp_path.size(),
+                    "%zd.sancov.map.tmp", internal_getpid());
+  uptr map_fd = OpenFile(tmp_path.data(), true);
+  if (internal_iserror(map_fd)) {
+    Report(" Coverage: failed to open %s for writing\n", tmp_path.data());
+    Die();
+  }
+
+  InternalScopedBuffer<char> modules_data(kMaxNumberOfModules *
+                                          sizeof(LoadedModule));
+  LoadedModule *modules = (LoadedModule *)modules_data.data();
+  CHECK(modules);
+  int n_modules = GetListOfModules(modules, kMaxNumberOfModules,
+                                   /* filter */ 0);
+
+  InternalScopedString line(4096);
+  line.append("%d\n", sizeof(uptr) * 8);
+  uptr res = internal_write(map_fd, line.data(), line.length());
+  if (internal_iserror(res, &err)) {
+    Printf("sancov.map write failed: %d\n", err);
+    Die();
+  }
+  line.clear();
+
+  for (int i = 0; i < n_modules; ++i) {
+    char *module_name = StripModuleName(modules[i].full_name());
+    for (unsigned j = 0; j < modules[i].n_ranges(); ++j) {
+      line.append("%zx %zx %zx %s\n", modules[i].address_range_start(j),
+                  modules[i].address_range_end(j), modules[i].base_address(),
+                  module_name);
+      res = internal_write(map_fd, line.data(), line.length());
+      if (internal_iserror(res, &err)) {
+        Printf("sancov.map write failed: %d\n", err);
+        Die();
+      }
+      line.clear();
+    }
+    InternalFree(module_name);
+  }
+
+  internal_close(map_fd);
+
+  InternalScopedString path(64);
+  internal_snprintf((char *)path.data(), path.size(), "%zd.sancov.map",
+                    internal_getpid());
+  res = internal_rename(tmp_path.data(), path.data());
+  if (internal_iserror(res, &err)) {
+    Printf("sancov.map rename failed: %d\n", err);
+    Die();
+  }
+}
+
+}  // namespace __sanitizer

compiler-rt/lib/sanitizer_common/sanitizer_flags.cc

@@ -55,6 +55,7 @@ void SetCommonFlagsDefaults(CommonFlags *f) {
   f->legacy_pthread_cond = false;
   f->intercept_tls_get_addr = false;
   f->coverage = false;
+  f->coverage_direct = false;
   f->full_address_space = false;
 }
 
@@ -127,6 +128,10 @@ void ParseCommonFlagsFromString(CommonFlags *f, const char *str) {
   ParseFlag(str, &f->coverage, "coverage",
       "If set, coverage information will be dumped at program shutdown (if the "
       "coverage instrumentation was enabled at compile time).");
+  ParseFlag(str, &f->coverage_direct, "coverage_direct",
+            "If set, coverage information will be dumped directly to a memory "
+            "mapped file. This way data is not lost even if the process is "
+            "suddenly killed.");
   ParseFlag(str, &f->full_address_space, "full_address_space",
             "Sanitize complete address space; "
             "by default kernel area on 32-bit platforms will not be sanitized");

compiler-rt/lib/sanitizer_common/sanitizer_flags.h

@@ -54,6 +54,7 @@ struct CommonFlags {
   bool help;
   uptr mmap_limit_mb;
   bool coverage;
+  bool coverage_direct;
   bool full_address_space;
 };
 

compiler-rt/lib/sanitizer_common/sanitizer_libc.h

@@ -74,6 +74,7 @@ uptr internal_open(const char *filename, int flags, u32 mode);
 
 uptr internal_read(fd_t fd, void *buf, uptr count);
 uptr internal_write(fd_t fd, const void *buf, uptr count);
+uptr internal_ftruncate(fd_t fd, uptr size);
 
 // OS
 uptr internal_filesize(fd_t fd);  // -1 on error.
@@ -83,6 +84,7 @@ uptr internal_fstat(fd_t fd, void *buf);
 uptr internal_dup2(int oldfd, int newfd);
 uptr internal_readlink(const char *path, char *buf, uptr bufsize);
 uptr internal_unlink(const char *path);
+uptr internal_rename(const char *oldpath, const char *newpath);
 void NORETURN internal__exit(int exitcode);
 uptr internal_lseek(fd_t fd, OFF_T offset, int whence);
 

compiler-rt/lib/sanitizer_common/sanitizer_linux.cc

@@ -134,7 +134,7 @@ uptr internal_open(const char *filename, int flags, u32 mode) {
 
 uptr OpenFile(const char *filename, bool write) {
   return internal_open(filename,
-      write ? O_WRONLY | O_CREAT /*| O_CLOEXEC*/ : O_RDONLY, 0660);
+      write ? O_RDWR | O_CREAT /*| O_CLOEXEC*/ : O_RDONLY, 0660);
 }
 
 uptr internal_read(fd_t fd, void *buf, uptr count) {
@@ -151,6 +151,12 @@ uptr internal_write(fd_t fd, const void *buf, uptr count) {
   return res;
 }
 
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  sptr res;
+  HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd, size));
+  return res;
+}
+
 #if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && !SANITIZER_FREEBSD
 static void stat64_to_stat(struct stat64 *in, struct stat *out) {
   internal_memset(out, 0, sizeof(*out));
@@ -246,6 +252,15 @@ uptr internal_unlink(const char *path) {
 #endif
 }
 
+uptr internal_rename(const char *oldpath, const char *newpath) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
+                          (uptr)newpath);
+#else
+  return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
+#endif
+}
+
 uptr internal_sched_yield() {
   return internal_syscall(SYSCALL(sched_yield));
 }
@@ -390,20 +405,6 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
 }
 #endif  // SANITIZER_GO
 
-void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
-  // Some kinds of sandboxes may forbid filesystem access, so we won't be able
-  // to read the file mappings from /proc/self/maps. Luckily, neither the
-  // process will be able to load additional libraries, so it's fine to use the
-  // cached mappings.
-  MemoryMappingLayout::CacheMemoryMappings();
-  // Same for /proc/self/exe in the symbolizer.
-#if !SANITIZER_GO
-  if (Symbolizer *sym = Symbolizer::GetOrNull())
-    sym->PrepareForSandboxing();
-  CovPrepareForSandboxing(args);
-#endif
-}
-
 enum MutexState {
   MtxUnlocked = 0,
   MtxLocked = 1,

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc

@@ -22,6 +22,7 @@
 #include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 #include "sanitizer_atomic.h"
+#include "sanitizer_symbolizer.h"
 
 #include <dlfcn.h>
 #include <pthread.h>
@@ -529,6 +530,20 @@ void SetIndirectCallWrapper(uptr wrapper) {
   indirect_call_wrapper = wrapper;
 }
 
+void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
+  // Some kinds of sandboxes may forbid filesystem access, so we won't be able
+  // to read the file mappings from /proc/self/maps. Luckily, neither the
+  // process will be able to load additional libraries, so it's fine to use the
+  // cached mappings.
+  MemoryMappingLayout::CacheMemoryMappings();
+  // Same for /proc/self/exe in the symbolizer.
+#if !SANITIZER_GO
+  if (Symbolizer *sym = Symbolizer::GetOrNull())
+    sym->PrepareForSandboxing();
+  CovPrepareForSandboxing(args);
+#endif
+}
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX

compiler-rt/lib/sanitizer_common/sanitizer_mac.cc

@@ -131,6 +131,14 @@ int internal_fork() {
   return fork();
 }
 
+uptr internal_rename(const char *oldpath, const char *newpath) {
+  return rename(oldpath, newpath);
+}
+
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  return ftruncate(fd, size);
+}
+
 // ----------------- sanitizer_common.h
 bool FileExists(const char *filename) {
   struct stat st;

compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h

@@ -203,5 +203,6 @@
 #define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS
 #define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE SI_NOT_WINDOWS
 
 #endif  // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H

compiler-rt/lib/sanitizer_common/sanitizer_posix.cc

@@ -206,6 +206,17 @@ void *MapFileToMemory(const char *file_name, uptr *buff_size) {
   return internal_iserror(map) ? 0 : (void *)map;
 }
 
+void *MapWritableFileToMemory(void *addr, uptr size, uptr fd, uptr offset) {
+  uptr flags = MAP_SHARED;
+  if (addr) flags |= MAP_FIXED;
+  uptr p = internal_mmap(addr, size, PROT_READ | PROT_WRITE, flags, fd, offset);
+  if (internal_iserror(p)) {
+    Printf("could not map writable file (%zd, %zu, %zu): %zd\n", fd, offset,
+           size, p);
+    return 0;
+  }
+  return (void *)p;
+}
 
 static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
                                         uptr start2, uptr end2) {

compiler-rt/lib/sanitizer_common/scripts/sancov.py

@@ -6,6 +6,8 @@
 import array
 import struct
 import sys
+import bisect
+import os.path
 
 prog_name = "";
 
@@ -74,6 +76,63 @@ def Unpack(files):
   for f in files:
     UnpackOneFile(f)
 
+def UnpackOneRawFile(path, map_path):
+  mem_map = []
+  with open(map_path, mode="rt") as f_map:
+    print >> sys.stderr, "%s: reading map %s" % (prog_name, map_path)
+    bits = int(f_map.readline())
+    for line in f_map:
+      parts = line.rstrip().split()
+      assert len(parts) == 4
+      mem_map.append((int(parts[0], 16),
+                  int(parts[1], 16),
+                  int(parts[2], 16),
+                  parts[3]))
+  mem_map.sort(key=lambda m : m[0])
+  mem_map_keys = [m[0] for m in mem_map]
+
+  print mem_map
+  with open(path, mode="rb") as f:
+    print >> sys.stderr, "%s: unpacking %s" % (prog_name, path)
+
+    f.seek(0, 2)
+    size = f.tell()
+    f.seek(0, 0)
+    if bits == 64:
+      typecode = 'L'
+    else:
+      typecode = 'I'
+    pcs = array.array(typecode, f.read(size))
+    mem_map_pcs = [[] for i in range(0, len(mem_map))]
+
+    for pc in pcs:
+      if pc == 0: continue
+      map_idx = bisect.bisect(mem_map_keys, pc) - 1
+      (start, end, base, module_path) = mem_map[map_idx]
+      print pc
+      print start, end, base, module_path
+      assert pc >= start
+      if pc >= end:
+        print >> sys.stderr, "warning: %s: pc %x outside of any known mapping" % (prog_name, pc)
+        continue
+      mem_map_pcs[map_idx].append(pc - base)
+
+    for ((start, end, base, module_path), pc_list) in zip(mem_map, mem_map_pcs):
+      if len(pc_list) == 0: continue
+      assert path.endswith('.sancov.raw')
+      dst_path = module_path + '.' + os.path.basename(path)[:-4]
+      print "writing %d PCs to %s" % (len(pc_list), dst_path)
+      arr = array.array('I')
+      arr.fromlist(sorted(pc_list))
+      with open(dst_path, 'ab') as f2:
+        arr.tofile(f2)
+
+def RawUnpack(files):
+  for f in files:
+    if not f.endswith('.sancov.raw'):
+      raise Exception('Unexpected raw file name %s' % f)
+    f_map = f[:-3] + 'map'
+    UnpackOneRawFile(f, f_map)
 
 if __name__ == '__main__':
   prog_name = sys.argv[0]
@@ -85,5 +144,7 @@ if __name__ == '__main__':
     MergeAndPrint(sys.argv[2:])
   elif sys.argv[1] == "unpack":
     Unpack(sys.argv[2:])
+  elif sys.argv[1] == "rawunpack":
+    RawUnpack(sys.argv[2:])
   else:
     Usage()

compiler-rt/lib/tsan/rtl/tsan_interceptors.cc

@@ -207,7 +207,7 @@ ScopedInterceptor::~ScopedInterceptor() {
     if (REAL(func) == 0) { \
       Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
       Die(); \
-    } \
+    }                                                    \
     if (thr->ignore_interceptors || thr->in_ignored_lib) \
       return REAL(func)(__VA_ARGS__); \
 /**/
@@ -259,20 +259,6 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
   return res;
 }
 
-TSAN_INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
-  SCOPED_INTERCEPTOR_RAW(dlopen, filename, flag);
-  void *res = REAL(dlopen)(filename, flag);
-  libignore()->OnLibraryLoaded(filename);
-  return res;
-}
-
-TSAN_INTERCEPTOR(int, dlclose, void *handle) {
-  SCOPED_INTERCEPTOR_RAW(dlclose, handle);
-  int res = REAL(dlclose)(handle);
-  libignore()->OnLibraryUnloaded();
-  return res;
-}
-
 class AtExitContext {
  public:
   AtExitContext()
@@ -2021,6 +2007,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
   ctx = (void *)&_ctx;                                \
   (void) ctx;
 
+#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
+  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
+  TsanInterceptorContext _ctx = {thr, caller_pc, pc};     \
+  ctx = (void *)&_ctx;                                    \
+  (void) ctx;
+
 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
   Acquire(thr, pc, File2addr(path));                  \
   if (file) {                                         \
@@ -2034,6 +2026,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
     if (fd >= 0) FdClose(thr, pc, fd);           \
   }
 
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res)  \
+  libignore()->OnLibraryLoaded(filename)
+
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
+  libignore()->OnLibraryUnloaded()
+
 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
   FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
 
@@ -2376,8 +2374,6 @@ void InitializeInterceptors() {
 
   TSAN_INTERCEPT(fork);
   TSAN_INTERCEPT(vfork);
-  TSAN_INTERCEPT(dlopen);
-  TSAN_INTERCEPT(dlclose);
   TSAN_INTERCEPT(on_exit);
   TSAN_INTERCEPT(__cxa_atexit);
   TSAN_INTERCEPT(_exit);

compiler-rt/test/asan/TestCases/Linux/coverage-direct-large.cc

@@ -0,0 +1,45 @@
+// Test for direct coverage writing with lots of data.
+// Current implementation maps output file in chunks of 64K. This test overflows
+// 1 chunk.
+// RUN: %clangxx_asan -mllvm -asan-coverage=1 -O0 %s -o %t
+
+// RUN: rm -rf %T/coverage-direct-large
+
+// RUN: mkdir -p %T/coverage-direct-large/normal && cd %T/coverage-direct-large/normal
+// RUN: ASAN_OPTIONS=coverage=1:coverage_direct=0:verbosity=1 %run %t
+// RUN: %sancov print *.sancov >out.txt
+// RUN: cd ../..
+
+// RUN: mkdir -p %T/coverage-direct-large/direct && cd %T/coverage-direct-large/direct
+// RUN: ASAN_OPTIONS=coverage=1:coverage_direct=1:verbosity=1 %run %t
+// RUN: %sancov rawunpack *.sancov.raw
+// RUN: %sancov print *.sancov >out.txt
+// RUN: cd ../..
+
+// RUN: diff -u coverage-direct-large/normal/out.txt coverage-direct-large/direct/out.txt
+//
+// XFAIL: android
+
+#define F0(Q, x) Q(x)
+#define F1(Q, x)                                                          \
+  F0(Q, x##0) F0(Q, x##1) F0(Q, x##2) F0(Q, x##3) F0(Q, x##4) F0(Q, x##5) \
+      F0(Q, x##6) F0(Q, x##7) F0(Q, x##8) F0(Q, x##9)
+#define F2(Q, x)                                                          \
+  F1(Q, x##0) F1(Q, x##1) F1(Q, x##2) F1(Q, x##3) F1(Q, x##4) F1(Q, x##5) \
+      F1(Q, x##6) F1(Q, x##7) F1(Q, x##8) F1(Q, x##9)
+#define F3(Q, x)                                                          \
+  F2(Q, x##0) F2(Q, x##1) F2(Q, x##2) F2(Q, x##3) F2(Q, x##4) F2(Q, x##5) \
+      F2(Q, x##6) F2(Q, x##7) F2(Q, x##8) F2(Q, x##9)
+#define F4(Q, x)                                                          \
+  F3(Q, x##0) F3(Q, x##1) F3(Q, x##2) F3(Q, x##3) F3(Q, x##4) F3(Q, x##5) \
+      F3(Q, x##6) F3(Q, x##7) F3(Q, x##8) F3(Q, x##9)
+
+#define DECL(x) __attribute__((noinline)) void x() {}
+#define CALL(x) x();
+
+F4(DECL, f)
+
+int main(void) {
+  F4(CALL, f)
+  return 0;
+}

compiler-rt/test/asan/TestCases/Linux/coverage-direct.cc

@@ -0,0 +1,44 @@
+// Test for direct coverage writing with dlopen.
+// RUN: %clangxx_asan -mllvm -asan-coverage=1 -DSHARED %s -shared -o %T/libcoverage_direct_test_1.so -fPIC
+// RUN: %clangxx_asan -mllvm -asan-coverage=1 -DSO_DIR=\"%T\" %s -o %t
+
+// RUN: rm -rf %T/coverage-direct
+
+// RUN: mkdir -p %T/coverage-direct/normal && cd %T/coverage-direct/normal
+// RUN: ASAN_OPTIONS=coverage=1:coverage_direct=0:verbosity=1 %run %t
+// RUN: %sancov print *.sancov >out.txt
+// RUN: cd ../..
+
+// RUN: mkdir -p %T/coverage-direct/direct && cd %T/coverage-direct/direct
+// RUN: ASAN_OPTIONS=coverage=1:coverage_direct=1:verbosity=1 %run %t
+// RUN: %sancov rawunpack *.sancov.raw
+// RUN: %sancov print *.sancov >out.txt
+// RUN: cd ../..
+
+// RUN: diff -u coverage-direct/normal/out.txt coverage-direct/direct/out.txt
+//
+// XFAIL: android
+
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#ifdef SHARED
+extern "C" {
+void bar() { printf("bar\n"); }
+}
+#else
+
+int main(int argc, char **argv) {
+  fprintf(stderr, "PID: %d\n", getpid());
+  void *handle1 =
+      dlopen(SO_DIR "/libcoverage_direct_test_1.so", RTLD_LAZY);
+  assert(handle1);
+  void (*bar1)() = (void (*)())dlsym(handle1, "bar");
+  assert(bar1);
+  bar1();
+
+  return 0;
+}
+#endif