Re-land "[FileCollector] Add a method to add a whole directory and it contents."

Extend the FileCollector's API with addDirectory which adds a directory
and its contents to the VFS mapping.

Differential revision: https://reviews.llvm.org/D76671

llvm/include/llvm/Support/FileCollector.h

@@ -18,7 +18,7 @@
 #include <mutex>
 
 namespace llvm {
-
+class FileCollectorFileSystem;
 /// Collects files into a directory and generates a mapping that can be used by
 /// the VFS.
 class FileCollector {
@@ -26,9 +26,10 @@ public:
   FileCollector(std::string Root, std::string OverlayRoot);
 
   void addFile(const Twine &file);
+  void addDirectory(const Twine &Dir);
 
   /// Write the yaml mapping (for the VFS) to the given file.
-  std::error_code writeMapping(StringRef mapping_file);
+  std::error_code writeMapping(StringRef MappingFile);
 
   /// Copy the files into the root directory.
   ///
@@ -44,7 +45,7 @@ public:
                      std::shared_ptr<FileCollector> Collector);
 
 private:
-  void addFileImpl(StringRef SrcPath);
+  friend FileCollectorFileSystem;
 
   bool markAsSeen(StringRef Path) {
     if (Path.empty())
@@ -55,10 +56,19 @@ private:
   bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result);
 
   void addFileToMapping(StringRef VirtualPath, StringRef RealPath) {
-    VFSWriter.addFileMapping(VirtualPath, RealPath);
+    if (sys::fs::is_directory(VirtualPath))
+      VFSWriter.addDirectoryMapping(VirtualPath, RealPath);
+    else
+      VFSWriter.addFileMapping(VirtualPath, RealPath);
   }
 
 protected:
+  void addFileImpl(StringRef SrcPath);
+
+  llvm::vfs::directory_iterator
+  addDirectoryImpl(const llvm::Twine &Dir,
+                   IntrusiveRefCntPtr<vfs::FileSystem> FS, std::error_code &EC);
+
   /// Synchronizes adding files.
   std::mutex Mutex;
 

llvm/lib/Support/FileCollector.cpp

@@ -61,13 +61,19 @@ bool FileCollector::getRealPath(StringRef SrcPath,
   return true;
 }
 
-void FileCollector::addFile(const Twine &file) {
+void FileCollector::addFile(const Twine &File) {
   std::lock_guard<std::mutex> lock(Mutex);
-  std::string FileStr = file.str();
+  std::string FileStr = File.str();
   if (markAsSeen(FileStr))
     addFileImpl(FileStr);
 }
 
+void FileCollector::addDirectory(const Twine &Dir) {
+  assert(sys::fs::is_directory(Dir));
+  std::error_code EC;
+  addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC);
+}
+
 void FileCollector::addFileImpl(StringRef SrcPath) {
   // We need an absolute src path to append to the root.
   SmallString<256> AbsoluteSrc = SrcPath;
@@ -101,6 +107,27 @@ void FileCollector::addFileImpl(StringRef SrcPath) {
   addFileToMapping(VirtualPath, DstPath);
 }
 
+llvm::vfs::directory_iterator
+FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
+                                IntrusiveRefCntPtr<vfs::FileSystem> FS,
+                                std::error_code &EC) {
+  auto It = FS->dir_begin(Dir, EC);
+  if (EC)
+    return It;
+  addFile(Dir);
+  for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
+    if (It->type() == sys::fs::file_type::regular_file ||
+        It->type() == sys::fs::file_type::directory_file ||
+        It->type() == sys::fs::file_type::symlink_file) {
+      addFile(It->path());
+    }
+  }
+  if (EC)
+    return It;
+  // Return a new iterator.
+  return FS->dir_begin(Dir, EC);
+}
+
 /// Set the access and modification time for the given file from the given
 /// status object.
 static std::error_code
@@ -171,7 +198,7 @@ std::error_code FileCollector::copyFiles(bool StopOnError) {
   return {};
 }
 
-std::error_code FileCollector::writeMapping(StringRef mapping_file) {
+std::error_code FileCollector::writeMapping(StringRef MappingFile) {
   std::lock_guard<std::mutex> lock(Mutex);
 
   VFSWriter.setOverlayDir(OverlayRoot);
@@ -179,7 +206,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   VFSWriter.setUseExternalNames(false);
 
   std::error_code EC;
-  raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
+  raw_fd_ostream os(MappingFile, EC, sys::fs::OF_Text);
   if (EC)
     return EC;
 
@@ -188,7 +215,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   return {};
 }
 
-namespace {
+namespace llvm {
 
 class FileCollectorFileSystem : public vfs::FileSystem {
 public:
@@ -213,22 +240,7 @@ public:
 
   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
                                           std::error_code &EC) override {
-    auto It = FS->dir_begin(Dir, EC);
-    if (EC)
-      return It;
-    // Collect everything that's listed in case the user needs it.
-    Collector->addFile(Dir);
-    for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
-      if (It->type() == sys::fs::file_type::regular_file ||
-          It->type() == sys::fs::file_type::directory_file ||
-          It->type() == sys::fs::file_type::symlink_file) {
-        Collector->addFile(It->path());
-      }
-    }
-    if (EC)
-      return It;
-    // Return a new iterator.
-    return FS->dir_begin(Dir, EC);
+    return Collector->addDirectoryImpl(Dir, FS, EC);
   }
 
   std::error_code getRealPath(const Twine &Path,
@@ -259,7 +271,7 @@ private:
   std::shared_ptr<FileCollector> Collector;
 };
 
-} // end anonymous namespace
+} // namespace llvm
 
 IntrusiveRefCntPtr<vfs::FileSystem>
 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,

llvm/unittests/Support/FileCollectorTest.cpp

@@ -120,6 +120,41 @@ TEST(FileCollectorTest, addFile) {
   EXPECT_FALSE(FileCollector.hasSeen("/path/to/d"));
 }
 
+TEST(FileCollectorTest, addDirectory) {
+  ScopedDir file_root("file_root", true);
+
+  llvm::SmallString<128> aaa = file_root.Path;
+  llvm::sys::path::append(aaa, "aaa");
+  ScopedFile a(aaa.str());
+
+  llvm::SmallString<128> bbb = file_root.Path;
+  llvm::sys::path::append(bbb, "bbb");
+  ScopedFile b(bbb.str());
+
+  llvm::SmallString<128> ccc = file_root.Path;
+  llvm::sys::path::append(ccc, "ccc");
+  ScopedFile c(ccc.str());
+
+  std::string root_fs = std::string(file_root.Path.str());
+  TestingFileCollector FileCollector(root_fs, root_fs);
+
+  FileCollector.addDirectory(file_root.Path);
+
+  // Make sure the root is correct.
+  EXPECT_EQ(FileCollector.Root, root_fs);
+
+  // Make sure we've seen all the added files.
+  EXPECT_TRUE(FileCollector.hasSeen(a.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(b.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(c.Path));
+
+  // Make sure we've only seen the added files.
+  llvm::SmallString<128> ddd = file_root.Path;
+  llvm::sys::path::append(ddd, "ddd");
+  ScopedFile d(ddd.str());
+  EXPECT_FALSE(FileCollector.hasSeen(d.Path));
+}
+
 TEST(FileCollectorTest, copyFiles) {
   ScopedDir file_root("file_root", true);
   ScopedFile a(file_root + "/aaa");