diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 8da1ead03cd2..ef81867b0e71 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -174,13 +174,13 @@ private:
   bool inLib = false;
 
   std::unique_ptr<BitcodeCompiler> lto;
-  std::vector<InputFile *> files;
+  SmallVector<std::unique_ptr<InputFile>, 0> files, ltoObjectFiles;
 
 public:
   // See InputFile::groupId.
   uint32_t nextGroupId;
   bool isInGroup;
-  InputFile *armCmseImpLib = nullptr;
+  std::unique_ptr<InputFile> armCmseImpLib;
   SmallVector<std::pair<StringRef, unsigned>, 0> archiveFiles;
 };
 
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 6e20d9a8527a..2592877203bf 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -231,8 +231,8 @@ bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
       IRObjectFile::findBitcodeInMemBuffer(mb);
   if (errorToBool(fatLTOData.takeError()))
     return false;
-  files.push_back(
-      make<BitcodeFile>(ctx, *fatLTOData, archiveName, offsetInArchive, lazy));
+  files.push_back(std::make_unique<BitcodeFile>(ctx, *fatLTOData, archiveName,
+                                                offsetInArchive, lazy));
   return true;
 }
 
@@ -246,7 +246,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
   MemoryBufferRef mbref = *buffer;
 
   if (ctx.arg.formatBinary) {
-    files.push_back(make<BinaryFile>(ctx, mbref));
+    files.push_back(std::make_unique<BinaryFile>(ctx, mbref));
     return;
   }
 
@@ -259,8 +259,8 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
     if (inWholeArchive) {
       for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
         if (isBitcode(p.first))
-          files.push_back(
-              make<BitcodeFile>(ctx, p.first, path, p.second, false));
+          files.push_back(std::make_unique<BitcodeFile>(ctx, p.first, path,
+                                                        p.second, false));
         else if (!tryAddFatLTOFile(p.first, path, p.second, false))
           files.push_back(createObjFile(ctx, p.first, path));
       }
@@ -288,7 +288,8 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
         if (!tryAddFatLTOFile(p.first, path, p.second, true))
           files.push_back(createObjFile(ctx, p.first, path, true));
       } else if (magic == file_magic::bitcode)
-        files.push_back(make<BitcodeFile>(ctx, p.first, path, p.second, true));
+        files.push_back(
+            std::make_unique<BitcodeFile>(ctx, p.first, path, p.second, true));
       else
         Warn(ctx) << path << ": archive member '"
                   << p.first.getBufferIdentifier()
@@ -309,14 +310,14 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
     // the directory part is ignored. Note that path may be a temporary and
     // cannot be stored into SharedFile::soName.
     path = mbref.getBufferIdentifier();
-    auto *f =
-        make<SharedFile>(ctx, mbref, withLOption ? path::filename(path) : path);
+    auto f = std::make_unique<SharedFile>(
+        ctx, mbref, withLOption ? path::filename(path) : path);
     f->init();
-    files.push_back(f);
+    files.push_back(std::move(f));
     return;
   }
   case file_magic::bitcode:
-    files.push_back(make<BitcodeFile>(ctx, mbref, "", 0, inLib));
+    files.push_back(std::make_unique<BitcodeFile>(ctx, mbref, "", 0, inLib));
     break;
   case file_magic::elf_relocatable:
     if (!tryAddFatLTOFile(mbref, "", 0, inLib))
@@ -2040,7 +2041,7 @@ void LinkerDriver::inferMachineType() {
     return;
 
   bool inferred = false;
-  for (InputFile *f : files) {
+  for (auto &f : files) {
     if (f->ekind == ELFNoneKind)
       continue;
     if (!inferred) {
@@ -2530,8 +2531,9 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
   if (!ctx.bitcodeFiles.empty())
     markBuffersAsDontNeed(ctx, skipLinkedOutput);
 
-  for (InputFile *file : lto->compile()) {
-    auto *obj = cast<ObjFile<ELFT>>(file);
+  ltoObjectFiles = lto->compile();
+  for (auto &file : ltoObjectFiles) {
+    auto *obj = cast<ObjFile<ELFT>>(file.get());
     obj->parse(/*ignoreComdats=*/true);
 
     // Parse '@' in symbol names for non-relocatable output.
@@ -3039,10 +3041,9 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   auto newInputFiles = ArrayRef(ctx.driver.files).slice(numInputFilesBeforeLTO);
   if (!newInputFiles.empty()) {
     DenseSet<StringRef> oldFilenames;
-    for (InputFile *f :
-         ArrayRef(ctx.driver.files).slice(0, numInputFilesBeforeLTO))
+    for (auto &f : ArrayRef(ctx.driver.files).slice(0, numInputFilesBeforeLTO))
       oldFilenames.insert(f->getName());
-    for (InputFile *newFile : newInputFiles)
+    for (auto &newFile : newInputFiles)
       if (!oldFilenames.contains(newFile->getName()))
         Err(ctx) << "input file '" << newFile->getName() << "' added after LTO";
   }
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 224912706f20..3a0ae43b813f 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -216,6 +216,8 @@ InputFile::InputFile(Ctx &ctx, Kind k, MemoryBufferRef m)
     ++ctx.driver.nextGroupId;
 }
 
+InputFile::~InputFile() {}
+
 std::optional<MemoryBufferRef> elf::readFile(Ctx &ctx, StringRef path) {
   llvm::TimeTraceScope timeScope("Load input files", path);
 
@@ -345,19 +347,22 @@ extern template void ObjFile<ELF64LE>::importCmseSymbols();
 extern template void ObjFile<ELF64BE>::importCmseSymbols();
 
 template <class ELFT>
-static void doParseFiles(Ctx &ctx, const std::vector<InputFile *> &files) {
+static void
+doParseFiles(Ctx &ctx,
+             const SmallVector<std::unique_ptr<InputFile>, 0> &files) {
   // Add all files to the symbol table. This will add almost all symbols that we
   // need to the symbol table. This process might add files to the link due to
   // addDependentLibrary.
   for (size_t i = 0; i < files.size(); ++i) {
     llvm::TimeTraceScope timeScope("Parse input files", files[i]->getName());
-    doParseFile<ELFT>(ctx, files[i]);
+    doParseFile<ELFT>(ctx, files[i].get());
   }
   if (ctx.driver.armCmseImpLib)
     cast<ObjFile<ELFT>>(*ctx.driver.armCmseImpLib).importCmseSymbols();
 }
 
-void elf::parseFiles(Ctx &ctx, const std::vector<InputFile *> &files) {
+void elf::parseFiles(Ctx &ctx,
+                     const SmallVector<std::unique_ptr<InputFile>, 0> &files) {
   llvm::TimeTraceScope timeScope("Parse input files");
   invokeELFT(doParseFiles, ctx, files);
 }
@@ -1878,21 +1883,22 @@ InputFile *elf::createInternalFile(Ctx &ctx, StringRef name) {
   return file;
 }
 
-ELFFileBase *elf::createObjFile(Ctx &ctx, MemoryBufferRef mb,
-                                StringRef archiveName, bool lazy) {
-  ELFFileBase *f;
+std::unique_ptr<ELFFileBase> elf::createObjFile(Ctx &ctx, MemoryBufferRef mb,
+                                                StringRef archiveName,
+                                                bool lazy) {
+  std::unique_ptr<ELFFileBase> f;
   switch (getELFKind(ctx, mb, archiveName)) {
   case ELF32LEKind:
-    f = make<ObjFile<ELF32LE>>(ctx, ELF32LEKind, mb, archiveName);
+    f = std::make_unique<ObjFile<ELF32LE>>(ctx, ELF32LEKind, mb, archiveName);
     break;
   case ELF32BEKind:
-    f = make<ObjFile<ELF32BE>>(ctx, ELF32BEKind, mb, archiveName);
+    f = std::make_unique<ObjFile<ELF32BE>>(ctx, ELF32BEKind, mb, archiveName);
     break;
   case ELF64LEKind:
-    f = make<ObjFile<ELF64LE>>(ctx, ELF64LEKind, mb, archiveName);
+    f = std::make_unique<ObjFile<ELF64LE>>(ctx, ELF64LEKind, mb, archiveName);
     break;
   case ELF64BEKind:
-    f = make<ObjFile<ELF64BE>>(ctx, ELF64BEKind, mb, archiveName);
+    f = std::make_unique<ObjFile<ELF64BE>>(ctx, ELF64BEKind, mb, archiveName);
     break;
   default:
     llvm_unreachable("getELFKind");
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index a30d8a5ca7d6..9161fa13ad98 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -44,7 +44,7 @@ std::optional<MemoryBufferRef> readFile(Ctx &, StringRef path);
 
 // Add symbols in File to the symbol table.
 void parseFile(Ctx &, InputFile *file);
-void parseFiles(Ctx &, const std::vector<InputFile *> &files);
+void parseFiles(Ctx &, const SmallVector<std::unique_ptr<InputFile>, 0> &);
 
 // The root class of input files.
 class InputFile {
@@ -66,6 +66,7 @@ public:
   };
 
   InputFile(Ctx &, Kind k, MemoryBufferRef m);
+  virtual ~InputFile();
   Kind kind() const { return fileKind; }
 
   bool isElf() const {
@@ -380,8 +381,9 @@ public:
 };
 
 InputFile *createInternalFile(Ctx &, StringRef name);
-ELFFileBase *createObjFile(Ctx &, MemoryBufferRef mb,
-                           StringRef archiveName = "", bool lazy = false);
+std::unique_ptr<ELFFileBase> createObjFile(Ctx &, MemoryBufferRef mb,
+                                           StringRef archiveName = "",
+                                           bool lazy = false);
 
 std::string replaceThinLTOSuffix(Ctx &, StringRef path);
 
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 4f57463020b1..5ed69c8c3bef 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -310,7 +310,7 @@ static void thinLTOCreateEmptyIndexFiles(Ctx &ctx) {
 
 // Merge all the bitcode files we have seen, codegen the result
 // and return the resulting ObjectFile(s).
-std::vector<InputFile *> BitcodeCompiler::compile() {
+SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);
@@ -373,7 +373,7 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
   }
 
   bool savePrelink = ctx.arg.saveTempsArgs.contains("prelink");
-  std::vector<InputFile *> ret;
+  SmallVector<std::unique_ptr<InputFile>, 0> ret;
   const char *ext = ctx.arg.ltoEmitAsm ? ".s" : ".o";
   for (unsigned i = 0; i != maxTasks; ++i) {
     StringRef bitcodeFilePath;
diff --git a/lld/ELF/LTO.h b/lld/ELF/LTO.h
index 6d60c5b65e9b..acf3bcff7f2f 100644
--- a/lld/ELF/LTO.h
+++ b/lld/ELF/LTO.h
@@ -42,7 +42,7 @@ public:
   ~BitcodeCompiler();
 
   void add(BitcodeFile &f);
-  std::vector<InputFile *> compile();
+  SmallVector<std::unique_ptr<InputFile>, 0> compile();
 
 private:
   Ctx &ctx;