[ClangRepl] Reland Semanic Code Completion (#75556)

This patch contains changes from
002d471a4a3cd8b429e4ca7c84fd54a642e50e4c, in
addition to a bug fix that added a virtual destructor to
`CompletionContextHandler`

The original changes in the orginal commit piggybacks on clang's
semantic modules to enable semantic completion.  In particular, we use
`CodeCompletionContext` to differentiate two types of code completion.
We also
extract the relevant type information from it.
This commit is contained in:
Fred Fu 2023-12-19 15:53:02 -05:00 committed by GitHub
parent edbd034248
commit 35b366ace7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 444 additions and 50 deletions

View File

@ -23,8 +23,27 @@ namespace clang {
class CodeCompletionResult;
class CompilerInstance;
void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
unsigned Line, unsigned Col, const CompilerInstance *ParentCI,
std::vector<std::string> &CCResults);
struct ReplCodeCompleter {
ReplCodeCompleter() = default;
std::string Prefix;
/// \param InterpCI [in] The compiler instance that is used to trigger code
/// completion
/// \param Content [in] The string where code completion is triggered.
/// \param Line [in] The line number of the code completion point.
/// \param Col [in] The column number of the code completion point.
/// \param ParentCI [in] The running interpreter compiler instance that
/// provides ASTContexts.
/// \param CCResults [out] The completion results.
void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
unsigned Line, unsigned Col,
const CompilerInstance *ParentCI,
std::vector<std::string> &CCResults);
};
} // namespace clang
#endif

View File

@ -101,6 +101,7 @@ public:
const ASTContext &getASTContext() const;
ASTContext &getASTContext();
const CompilerInstance *getCompilerInstance() const;
CompilerInstance *getCompilerInstance();
llvm::Expected<llvm::orc::LLJIT &> getExecutionEngine();
llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);

View File

@ -12,6 +12,7 @@
#include "clang/Interpreter/CodeCompletion.h"
#include "clang/AST/ASTImporter.h"
#include "clang/AST/DeclLookups.h"
#include "clang/AST/DeclarationName.h"
#include "clang/AST/ExternalASTSource.h"
#include "clang/Basic/IdentifierTable.h"
@ -23,6 +24,8 @@
#include "clang/Sema/CodeCompleteConsumer.h"
#include "clang/Sema/CodeCompleteOptions.h"
#include "clang/Sema/Sema.h"
#include "llvm/Support/Debug.h"
#define DEBUG_TYPE "REPLCC"
namespace clang {
@ -39,11 +42,15 @@ clang::CodeCompleteOptions getClangCompleteOpts() {
class ReplCompletionConsumer : public CodeCompleteConsumer {
public:
ReplCompletionConsumer(std::vector<std::string> &Results)
ReplCompletionConsumer(std::vector<std::string> &Results,
ReplCodeCompleter &CC)
: CodeCompleteConsumer(getClangCompleteOpts()),
CCAllocator(std::make_shared<GlobalCodeCompletionAllocator>()),
CCTUInfo(CCAllocator), Results(Results){};
CCTUInfo(CCAllocator), Results(Results), CC(CC) {}
// The entry of handling code completion. When the function is called, we
// create a `Context`-based handler (see classes defined below) to handle each
// completion result.
void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context,
CodeCompletionResult *InResults,
unsigned NumResults) final;
@ -56,26 +63,147 @@ private:
std::shared_ptr<GlobalCodeCompletionAllocator> CCAllocator;
CodeCompletionTUInfo CCTUInfo;
std::vector<std::string> &Results;
ReplCodeCompleter &CC;
};
/// The class CompletionContextHandler contains four interfaces, each of
/// which handles one type of completion result.
/// Its derived classes are used to create concrete handlers based on
/// \c CodeCompletionContext.
class CompletionContextHandler {
protected:
CodeCompletionContext CCC;
std::vector<std::string> &Results;
private:
Sema &S;
public:
CompletionContextHandler(Sema &S, CodeCompletionContext CCC,
std::vector<std::string> &Results)
: CCC(CCC), Results(Results), S(S) {}
virtual ~CompletionContextHandler() = default;
/// Converts a Declaration completion result to a completion string, and then
/// stores it in Results.
virtual void handleDeclaration(const CodeCompletionResult &Result) {
auto PreferredType = CCC.getPreferredType();
if (PreferredType.isNull()) {
Results.push_back(Result.Declaration->getName().str());
return;
}
if (auto *VD = dyn_cast<VarDecl>(Result.Declaration)) {
auto ArgumentType = VD->getType();
if (PreferredType->isReferenceType()) {
QualType RT = PreferredType->castAs<ReferenceType>()->getPointeeType();
Sema::ReferenceConversions RefConv;
Sema::ReferenceCompareResult RefRelationship =
S.CompareReferenceRelationship(SourceLocation(), RT, ArgumentType,
&RefConv);
switch (RefRelationship) {
case Sema::Ref_Compatible:
case Sema::Ref_Related:
Results.push_back(VD->getName().str());
break;
case Sema::Ref_Incompatible:
break;
}
} else if (S.Context.hasSameType(ArgumentType, PreferredType)) {
Results.push_back(VD->getName().str());
}
}
}
/// Converts a Keyword completion result to a completion string, and then
/// stores it in Results.
virtual void handleKeyword(const CodeCompletionResult &Result) {
auto Prefix = S.getPreprocessor().getCodeCompletionFilter();
// Add keyword to the completion results only if we are in a type-aware
// situation.
if (!CCC.getBaseType().isNull() || !CCC.getPreferredType().isNull())
return;
if (StringRef(Result.Keyword).startswith(Prefix))
Results.push_back(Result.Keyword);
}
/// Converts a Pattern completion result to a completion string, and then
/// stores it in Results.
virtual void handlePattern(const CodeCompletionResult &Result) {}
/// Converts a Macro completion result to a completion string, and then stores
/// it in Results.
virtual void handleMacro(const CodeCompletionResult &Result) {}
};
class DotMemberAccessHandler : public CompletionContextHandler {
public:
DotMemberAccessHandler(Sema &S, CodeCompletionContext CCC,
std::vector<std::string> &Results)
: CompletionContextHandler(S, CCC, Results) {}
void handleDeclaration(const CodeCompletionResult &Result) override {
auto *ID = Result.Declaration->getIdentifier();
if (!ID)
return;
if (!isa<CXXMethodDecl>(Result.Declaration))
return;
const auto *Fun = cast<CXXMethodDecl>(Result.Declaration);
if (Fun->getParent()->getCanonicalDecl() ==
CCC.getBaseType()->getAsCXXRecordDecl()->getCanonicalDecl()) {
LLVM_DEBUG(llvm::dbgs() << "[In HandleCodeCompleteDOT] Name : "
<< ID->getName() << "\n");
Results.push_back(ID->getName().str());
}
}
void handleKeyword(const CodeCompletionResult &Result) override {}
};
void ReplCompletionConsumer::ProcessCodeCompleteResults(
class Sema &S, CodeCompletionContext Context,
CodeCompletionResult *InResults, unsigned NumResults) {
for (unsigned I = 0; I < NumResults; ++I) {
auto Prefix = S.getPreprocessor().getCodeCompletionFilter();
CC.Prefix = Prefix;
std::unique_ptr<CompletionContextHandler> CCH;
// initialize fine-grained code completion handler based on the code
// completion context.
switch (Context.getKind()) {
case CodeCompletionContext::CCC_DotMemberAccess:
CCH.reset(new DotMemberAccessHandler(S, Context, this->Results));
break;
default:
CCH.reset(new CompletionContextHandler(S, Context, this->Results));
};
for (unsigned I = 0; I < NumResults; I++) {
auto &Result = InResults[I];
switch (Result.Kind) {
case CodeCompletionResult::RK_Declaration:
if (auto *ID = Result.Declaration->getIdentifier()) {
Results.push_back(ID->getName().str());
if (Result.Hidden) {
break;
}
if (!Result.Declaration->getDeclName().isIdentifier() ||
!Result.Declaration->getName().startswith(Prefix)) {
break;
}
CCH->handleDeclaration(Result);
break;
case CodeCompletionResult::RK_Keyword:
Results.push_back(Result.Keyword);
CCH->handleKeyword(Result);
break;
default:
case CodeCompletionResult::RK_Macro:
CCH->handleMacro(Result);
break;
case CodeCompletionResult::RK_Pattern:
CCH->handlePattern(Result);
break;
}
}
std::sort(Results.begin(), Results.end());
}
class IncrementalSyntaxOnlyAction : public SyntaxOnlyAction {
@ -118,6 +246,16 @@ void IncrementalSyntaxOnlyAction::ExecuteAction() {
CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage(
true);
// Load all external decls into current context. Under the hood, it calls
// ExternalSource::completeVisibleDeclsMap, which make all decls on the redecl
// chain visible.
//
// This is crucial to code completion on dot members, since a bound variable
// before "." would be otherwise treated out-of-scope.
//
// clang-repl> Foo f1;
// clang-repl> f1.<tab>
CI.getASTContext().getTranslationUnitDecl()->lookups();
SyntaxOnlyAction::ExecuteAction();
}
@ -134,6 +272,7 @@ ExternalSource::ExternalSource(ASTContext &ChildASTCtxt, FileManager &ChildFM,
bool ExternalSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
DeclarationName Name) {
IdentifierTable &ParentIdTable = ParentASTCtxt.Idents;
auto ParentDeclName =
@ -159,29 +298,67 @@ void ExternalSource::completeVisibleDeclsMap(
for (auto *DeclCtxt = ParentTUDeclCtxt; DeclCtxt != nullptr;
DeclCtxt = DeclCtxt->getPreviousDecl()) {
for (auto &IDeclContext : DeclCtxt->decls()) {
if (NamedDecl *Decl = llvm::dyn_cast<NamedDecl>(IDeclContext)) {
if (auto DeclOrErr = Importer->Import(Decl)) {
if (NamedDecl *importedNamedDecl =
llvm::dyn_cast<NamedDecl>(*DeclOrErr)) {
SetExternalVisibleDeclsForName(ChildDeclContext,
importedNamedDecl->getDeclName(),
importedNamedDecl);
}
if (!llvm::isa<NamedDecl>(IDeclContext))
continue;
} else {
llvm::consumeError(DeclOrErr.takeError());
}
NamedDecl *Decl = llvm::cast<NamedDecl>(IDeclContext);
auto DeclOrErr = Importer->Import(Decl);
if (!DeclOrErr) {
// if an error happens, it usually means the decl has already been
// imported or the decl is a result of a failed import. But in our
// case, every import is fresh each time code completion is
// triggered. So Import usually doesn't fail. If it does, it just means
// the related decl can't be used in code completion and we can safely
// drop it.
llvm::consumeError(DeclOrErr.takeError());
continue;
}
if (!llvm::isa<NamedDecl>(*DeclOrErr))
continue;
NamedDecl *importedNamedDecl = llvm::cast<NamedDecl>(*DeclOrErr);
SetExternalVisibleDeclsForName(ChildDeclContext,
importedNamedDecl->getDeclName(),
importedNamedDecl);
if (!llvm::isa<CXXRecordDecl>(importedNamedDecl))
continue;
auto *Record = llvm::cast<CXXRecordDecl>(importedNamedDecl);
if (auto Err = Importer->ImportDefinition(Decl)) {
// the same as above
consumeError(std::move(Err));
continue;
}
Record->setHasLoadedFieldsFromExternalStorage(true);
LLVM_DEBUG(llvm::dbgs()
<< "\nCXXRecrod : " << Record->getName() << " size(methods): "
<< std::distance(Record->method_begin(), Record->method_end())
<< " has def?: " << Record->hasDefinition()
<< " # (methods): "
<< std::distance(Record->getDefinition()->method_begin(),
Record->getDefinition()->method_end())
<< "\n");
for (auto *Meth : Record->methods())
SetExternalVisibleDeclsForName(ChildDeclContext, Meth->getDeclName(),
Meth);
}
ChildDeclContext->setHasExternalLexicalStorage(false);
}
}
void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
unsigned Line, unsigned Col, const CompilerInstance *ParentCI,
std::vector<std::string> &CCResults) {
void ReplCodeCompleter::codeComplete(CompilerInstance *InterpCI,
llvm::StringRef Content, unsigned Line,
unsigned Col,
const CompilerInstance *ParentCI,
std::vector<std::string> &CCResults) {
auto DiagOpts = DiagnosticOptions();
auto consumer = ReplCompletionConsumer(CCResults);
auto consumer = ReplCompletionConsumer(CCResults, *this);
auto diag = InterpCI->getDiagnosticsPtr();
std::unique_ptr<ASTUnit> AU(ASTUnit::LoadFromCompilerInvocationAction(

View File

@ -319,6 +319,10 @@ const CompilerInstance *Interpreter::getCompilerInstance() const {
return IncrParser->getCI();
}
CompilerInstance *Interpreter::getCompilerInstance() {
return IncrParser->getCI();
}
llvm::Expected<llvm::orc::LLJIT &> Interpreter::getExecutionEngine() {
if (!IncrExecutor) {
if (auto Err = CreateExecutor())

View File

@ -15,6 +15,8 @@
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Interpreter/CodeCompletion.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/Sema.h"
#include "llvm/ExecutionEngine/Orc/LLJIT.h"
#include "llvm/LineEditor/LineEditor.h"
@ -123,22 +125,14 @@ ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos,
return {};
}
codeComplete(
const_cast<clang::CompilerInstance *>((*Interp)->getCompilerInstance()),
Buffer, Lines, Pos + 1, MainInterp.getCompilerInstance(), Results);
size_t space_pos = Buffer.rfind(" ");
llvm::StringRef Prefix;
if (space_pos == llvm::StringRef::npos) {
Prefix = Buffer;
} else {
Prefix = Buffer.substr(space_pos + 1);
}
auto *MainCI = (*Interp)->getCompilerInstance();
auto CC = clang::ReplCodeCompleter();
CC.codeComplete(MainCI, Buffer, Lines, Pos + 1,
MainInterp.getCompilerInstance(), Results);
for (auto c : Results) {
if (c.find(Prefix) == 0)
Comps.push_back(llvm::LineEditor::Completion(c.substr(Prefix.size()), c));
if (c.find(CC.Prefix) == 0)
Comps.push_back(
llvm::LineEditor::Completion(c.substr(CC.Prefix.size()), c));
}
return Comps;
}

View File

@ -1,7 +1,9 @@
#include "clang/Interpreter/CodeCompletion.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Interpreter/Interpreter.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Sema/CodeCompleteConsumer.h"
#include "clang/Sema/Sema.h"
#include "llvm/LineEditor/LineEditor.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_ostream.h"
@ -19,7 +21,7 @@ static std::unique_ptr<Interpreter> createInterpreter() {
}
static std::vector<std::string> runComp(clang::Interpreter &MainInterp,
llvm::StringRef Prefix,
llvm::StringRef Input,
llvm::Error &ErrR) {
auto CI = CB.CreateCpp();
if (auto Err = CI.takeError()) {
@ -37,16 +39,14 @@ static std::vector<std::string> runComp(clang::Interpreter &MainInterp,
std::vector<std::string> Results;
std::vector<std::string> Comps;
codeComplete(
const_cast<clang::CompilerInstance *>((*Interp)->getCompilerInstance()),
Prefix, /* Lines */ 1, Prefix.size(), MainInterp.getCompilerInstance(),
Results);
auto *MainCI = (*Interp)->getCompilerInstance();
auto CC = ReplCodeCompleter();
CC.codeComplete(MainCI, Input, /* Lines */ 1, Input.size() + 1,
MainInterp.getCompilerInstance(), Results);
for (auto Res : Results)
if (Res.find(Prefix) == 0)
if (Res.find(CC.Prefix) == 0)
Comps.push_back(Res);
return Comps;
}
@ -62,8 +62,9 @@ TEST(CodeCompletionTest, Sanity) {
}
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, "f", Err);
EXPECT_EQ((size_t)2, comps.size()); // foo and float
EXPECT_EQ(comps[0], std::string("foo"));
EXPECT_EQ((size_t)2, comps.size()); // float and foo
EXPECT_EQ(comps[0], std::string("float"));
EXPECT_EQ(comps[1], std::string("foo"));
EXPECT_EQ((bool)Err, false);
}
@ -110,4 +111,202 @@ TEST(CodeCompletionTest, CompFunDeclsNoError) {
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, TypedDirected) {
auto Interp = createInterpreter();
if (auto R = Interp->ParseAndExecute("int application = 12;")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("char apple = '2';")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("void add(int &SomeInt){}")) {
consumeError(std::move(R));
return;
}
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("add("), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ((bool)Err, false);
}
if (auto R = Interp->ParseAndExecute("int banana = 42;")) {
consumeError(std::move(R));
return;
}
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("add("), Err);
EXPECT_EQ((size_t)2, comps.size());
EXPECT_EQ(comps[0], "application");
EXPECT_EQ(comps[1], "banana");
EXPECT_EQ((bool)Err, false);
}
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("add(b"), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], "banana");
EXPECT_EQ((bool)Err, false);
}
}
TEST(CodeCompletionTest, SanityClasses) {
auto Interp = createInterpreter();
if (auto R = Interp->ParseAndExecute("struct Apple{};")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("void takeApple(Apple &a1){}")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("Apple a1;")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("void takeAppleCopy(Apple a1){}")) {
consumeError(std::move(R));
return;
}
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, "takeApple(", Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("a1"));
EXPECT_EQ((bool)Err, false);
}
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("takeAppleCopy("), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("a1"));
EXPECT_EQ((bool)Err, false);
}
}
TEST(CodeCompletionTest, SubClassing) {
auto Interp = createInterpreter();
if (auto R = Interp->ParseAndExecute("struct Fruit {};")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("struct Apple : Fruit{};")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("void takeFruit(Fruit &f){}")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("Apple a1;")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("Fruit f1;")) {
consumeError(std::move(R));
return;
}
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("takeFruit("), Err);
EXPECT_EQ((size_t)2, comps.size());
EXPECT_EQ(comps[0], std::string("a1"));
EXPECT_EQ(comps[1], std::string("f1"));
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, MultipleArguments) {
auto Interp = createInterpreter();
if (auto R = Interp->ParseAndExecute("int foo = 42;")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("char fowl = 'A';")) {
consumeError(std::move(R));
return;
}
if (auto R = Interp->ParseAndExecute("void takeTwo(int &a, char b){}")) {
consumeError(std::move(R));
return;
}
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("takeTwo(foo, "), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("fowl"));
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, Methods) {
auto Interp = createInterpreter();
cantFail(Interp->ParseAndExecute(
"struct Foo{int add(int a){return 42;} int par(int b){return 42;}};"));
cantFail(Interp->ParseAndExecute("Foo f1;"));
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("f1."), Err);
EXPECT_EQ((size_t)2, comps.size());
EXPECT_EQ(comps[0], std::string("add"));
EXPECT_EQ(comps[1], std::string("par"));
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, MethodsInvocations) {
auto Interp = createInterpreter();
cantFail(Interp->ParseAndExecute(
"struct Foo{int add(int a){return 42;} int par(int b){return 42;}};"));
cantFail(Interp->ParseAndExecute("Foo f1;"));
cantFail(Interp->ParseAndExecute("int a = 84;"));
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("f1.add("), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("a"));
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, NestedInvocations) {
auto Interp = createInterpreter();
cantFail(Interp->ParseAndExecute(
"struct Foo{int add(int a){return 42;} int par(int b){return 42;}};"));
cantFail(Interp->ParseAndExecute("Foo f1;"));
cantFail(Interp->ParseAndExecute("int a = 84;"));
cantFail(Interp->ParseAndExecute("int plus(int a, int b) { return a + b; }"));
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("plus(42, f1.add("), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("a"));
EXPECT_EQ((bool)Err, false);
}
TEST(CodeCompletionTest, TemplateFunctions) {
auto Interp = createInterpreter();
cantFail(
Interp->ParseAndExecute("template <typename T> T id(T a) { return a;} "));
cantFail(Interp->ParseAndExecute("int apple = 84;"));
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("id<int>("), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("apple"));
EXPECT_EQ((bool)Err, false);
}
cantFail(Interp->ParseAndExecute(
"template <typename T> T pickFirst(T a, T b) { return a;} "));
cantFail(Interp->ParseAndExecute("char pear = '4';"));
{
auto Err = llvm::Error::success();
auto comps = runComp(*Interp, std::string("pickFirst(apple, "), Err);
EXPECT_EQ((size_t)1, comps.size());
EXPECT_EQ(comps[0], std::string("apple"));
EXPECT_EQ((bool)Err, false);
}
}
} // anonymous namespace