From 3dd541487608de5160ba4abc66ca13307db22e8a Mon Sep 17 00:00:00 2001 From: SahilPatidar Date: Mon, 13 Apr 2026 11:46:12 +0530 Subject: [PATCH] [Cling] Refactor CIFactory and add incremental action support --- core/dictgen/src/LinkdefReader.cxx | 62 ++- .../include/cling/Interpreter/CIFactory.h | 2 + .../include/cling/Interpreter/Interpreter.h | 9 + .../cling/lib/Interpreter/CIFactory.cpp | 500 ++++++------------ .../cling/lib/Interpreter/DeclCollector.cpp | 4 +- .../cling/lib/Interpreter/DeclCollector.h | 4 +- .../cling/lib/Interpreter/IncrementalAction.h | 96 ++++ .../lib/Interpreter/IncrementalParser.cpp | 313 +++++++---- .../cling/lib/Interpreter/IncrementalParser.h | 13 +- .../cling/lib/Interpreter/Interpreter.cpp | 48 +- .../lib/Interpreter/InterpreterCallbacks.cpp | 13 +- .../include/clang/Frontend/CompilerInstance.h | 26 + .../clang/lib/CodeGen/CodeGenAction.cpp | 12 +- .../clang/lib/Frontend/CompilerInstance.cpp | 3 +- .../clang/lib/Frontend/FrontendAction.cpp | 12 +- 15 files changed, 632 insertions(+), 485 deletions(-) create mode 100644 interpreter/cling/lib/Interpreter/IncrementalAction.h diff --git a/core/dictgen/src/LinkdefReader.cxx b/core/dictgen/src/LinkdefReader.cxx index 0cb4ebdca6793..584e589db3e81 100644 --- a/core/dictgen/src/LinkdefReader.cxx +++ b/core/dictgen/src/LinkdefReader.cxx @@ -36,8 +36,10 @@ #include "clang/AST/ASTContext.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" #include "clang/Lex/Pragma.h" #include "cling/Interpreter/CIFactory.h" @@ -1056,26 +1058,60 @@ bool LinkdefReader::Parse(SelectionRules &sr, llvm::StringRef code, const std::v parserArgsC.push_back(parserArgs[i].c_str()); } + std::string Std = "-std=c++" + + std::to_string(cling::CIFactory::CxxStdCompiledWith()); + //parserArgsC.push_back("-E"); + parserArgsC.push_back("-fsyntax-only"); + parserArgsC.push_back("-U__CINT__"); + // parserArgsC.push_back(Std.c_str()); // Extract all #pragmas std::unique_ptr memBuf = llvm::MemoryBuffer::getMemBuffer(code, "CLING #pragma extraction"); clang::CompilerInstance *pragmaCI = cling::CIFactory::createCI(std::move(memBuf), parserArgsC.size(), &parserArgsC[0], llvmdir, std::nullopt /*Consumer*/, {} /*ModuleFileExtension*/, true /*OnlyLex*/); - clang::Preprocessor &PP = pragmaCI->getPreprocessor(); - clang::DiagnosticConsumer &DClient = pragmaCI->getDiagnosticClient(); - DClient.BeginSourceFile(pragmaCI->getLangOpts(), &PP); + struct PragmaCollectAction : public clang::SyntaxOnlyAction { + LinkdefReader &fLDR; + cling::Interpreter &fInterp; + PragmaCollectAction(LinkdefReader &ldr, cling::Interpreter &Interp) : fLDR(ldr), fInterp(Interp) {} + bool BeginSourceFileAction(clang::CompilerInstance &CI) override + { + if (CI.getLangOpts().Modules) + cling::CIFactory::collectModule(CI); + clang::Preprocessor &PP = CI.getPreprocessor(); + // Attach the handlers before we have started. PP takes the ownership. + PP.AddPragmaHandler(new PragmaLinkCollector(fLDR)); + PP.AddPragmaHandler(new PragmaCreateCollector(fLDR)); + PP.AddPragmaHandler(new PragmaExtraInclude(fLDR)); + PP.AddPragmaHandler(new PragmaIoReadInclude(fLDR)); + // cling::CIFactory::setupCompiler(&CI, fInterp.getOptions().CompilerOpts); + return clang::SyntaxOnlyAction::BeginSourceFileAction(CI); + } + + void ExecuteAction() override + { + clang::CompilerInstance &CI = getCompilerInstance(); + if (!CI.hasPreprocessor()) + return; + + // FIXME: Move the truncation aspect of this into Sema, we delayed this + // till here so the source manager would be initialized. + if (hasCodeCompletionSupport() && !CI.getFrontendOpts().CodeCompletionAt.FileName.empty()) + CI.createCodeCompletionConsumer(); - // FIXME: Reduce the code duplication across these collector classes. - PragmaLinkCollector pragmaLinkCollector(*this); - PragmaCreateCollector pragmaCreateCollector(*this); - PragmaExtraInclude pragmaExtraInclude(*this); - PragmaIoReadInclude pragmaIoReadInclude(*this); + // Use a code completion consumer? + clang::CodeCompleteConsumer *CompletionConsumer = nullptr; + if (CI.hasCodeCompletionConsumer()) + CompletionConsumer = &CI.getCodeCompletionConsumer(); - PP.AddPragmaHandler(&pragmaLinkCollector); - PP.AddPragmaHandler(&pragmaCreateCollector); - PP.AddPragmaHandler(&pragmaExtraInclude); - PP.AddPragmaHandler(&pragmaIoReadInclude); + if (!CI.hasSema()) + CI.createSema(getTranslationUnitKind(), CompletionConsumer); + } + + } Act(*this, fInterp); + + pragmaCI->ExecuteAction(Act); + clang::Preprocessor &PP = pragmaCI->getPreprocessor(); // Start parsing the specified input file. PP.EnterMainSourceFile(); @@ -1085,5 +1121,5 @@ bool LinkdefReader::Parse(SelectionRules &sr, llvm::StringRef code, const std::v } while (tok.isNot(clang::tok::annot_repl_input_end)); fSelectionRules = nullptr; - return 0 == DClient.getNumErrors(); + return 0 == pragmaCI->getDiagnosticClient().getNumErrors(); } diff --git a/interpreter/cling/include/cling/Interpreter/CIFactory.h b/interpreter/cling/include/cling/Interpreter/CIFactory.h index b587a3cf4858a..2c1395639a8de 100644 --- a/interpreter/cling/include/cling/Interpreter/CIFactory.h +++ b/interpreter/cling/include/cling/Interpreter/CIFactory.h @@ -33,6 +33,7 @@ namespace cling { using ModuleFileExtensions = std::vector>; + void collectModule(clang::CompilerInstance &CI); // TODO: Add overload that takes file not MemoryBuffer clang::CompilerInstance* @@ -48,6 +49,7 @@ namespace cling { std::optional> consumerOpt, const ModuleFileExtensions& moduleExtensions, bool OnlyLex = false); + unsigned CxxStdCompiledWith(); } // namespace CIFactory } // namespace cling #endif // CLING_CIFACTORY_H diff --git a/interpreter/cling/include/cling/Interpreter/Interpreter.h b/interpreter/cling/include/cling/Interpreter/Interpreter.h index 1fa0c0d23d83f..64a04cc6c574b 100644 --- a/interpreter/cling/include/cling/Interpreter/Interpreter.h +++ b/interpreter/cling/include/cling/Interpreter/Interpreter.h @@ -78,6 +78,7 @@ namespace cling { class CompilationOptions; class DynamicLibraryManager; class IncrementalCUDADeviceCompiler; + class IncrementalAction; class IncrementalExecutor; class IncrementalParser; class InterpreterCallbacks; @@ -184,6 +185,14 @@ namespace cling { /// std::unique_ptr TSCtx; + ///\brief compiler instance. + /// + std::unique_ptr m_CI; + + ///\brief The FrontendAction + /// + std::unique_ptr m_Act; + ///\brief Cling's execution engine - a well wrapped llvm execution engine. /// std::unique_ptr m_Executor; diff --git a/interpreter/cling/lib/Interpreter/CIFactory.cpp b/interpreter/cling/lib/Interpreter/CIFactory.cpp index 1442fb2961737..fa6b416153607 100644 --- a/interpreter/cling/lib/Interpreter/CIFactory.cpp +++ b/interpreter/cling/lib/Interpreter/CIFactory.cpp @@ -980,22 +980,30 @@ namespace { PreprocessorOptions& PPOpts = CI->getInvocation().getPreprocessorOpts(); SetPreprocessorFromBinary(PPOpts); - // Sanity check that clang delivered the language standard requested - if (CompilerOpts.DefaultLanguage(&LangOpts)) { - switch (CxxStdCompiledWith()) { - case 23: assert(LangOpts.CPlusPlus23 && "Language version mismatch"); - LLVM_FALLTHROUGH; - case 20: assert(LangOpts.CPlusPlus20 && "Language version mismatch"); - LLVM_FALLTHROUGH; - case 17: assert(LangOpts.CPlusPlus17 && "Language version mismatch"); - LLVM_FALLTHROUGH; - case 14: assert(LangOpts.CPlusPlus14 && "Language version mismatch"); - LLVM_FALLTHROUGH; - case 11: assert(LangOpts.CPlusPlus11 && "Language version mismatch"); - break; - default: assert(false && "You have an unhandled C++ standard!"); + const clang::FrontendOptions& FEOpts = CI->getFrontendOpts(); + if (!FEOpts.Inputs.size() || + FEOpts.Inputs[0].getKind().getFormat() != clang::InputKind::Precompiled) + // Sanity check that clang delivered the language standard requested + if (CompilerOpts.DefaultLanguage(&LangOpts)) { + switch (CxxStdCompiledWith()) { + case 23: + assert(LangOpts.CPlusPlus23 && "Language version mismatch"); + LLVM_FALLTHROUGH; + case 20: + assert(LangOpts.CPlusPlus20 && "Language version mismatch"); + LLVM_FALLTHROUGH; + case 17: + assert(LangOpts.CPlusPlus17 && "Language version mismatch"); + LLVM_FALLTHROUGH; + case 14: + assert(LangOpts.CPlusPlus14 && "Language version mismatch"); + LLVM_FALLTHROUGH; + case 11: + assert(LangOpts.CPlusPlus11 && "Language version mismatch"); + break; + default: assert(false && "You have an unhandled C++ standard!"); + } } - } PPOpts.addMacroDef("__CLING__"); if (LangOpts.CPlusPlus11 == 1) @@ -1058,234 +1066,6 @@ namespace { } }; - static void HandleProgramActions(CompilerInstance &CI) { - const clang::FrontendOptions& FrontendOpts = CI.getFrontendOpts(); - if (FrontendOpts.ProgramAction == clang::frontend::ModuleFileInfo) { - // Copied from FrontendActions.cpp - // FIXME: Remove when we switch to the new driver. - - class DumpModuleInfoListener : public ASTReaderListener { - llvm::raw_ostream &Out; - - public: - DumpModuleInfoListener(llvm::raw_ostream &OS) : Out(OS) { } - -#define DUMP_BOOLEAN(Value, Text) \ - Out.indent(4) << Text << ": " << (Value? "Yes" : "No") << "\n" - - bool ReadFullVersionInformation(StringRef FullVersion) override { - Out.indent(2) - << "Generated by " - << (FullVersion == getClangFullRepositoryVersion()? "this" - : "a different") - << " Clang: " << FullVersion << "\n"; - return ASTReaderListener::ReadFullVersionInformation(FullVersion); - } - - void ReadModuleName(StringRef ModuleName) override { - Out.indent(2) << "Module name: " << ModuleName << "\n"; - } - void ReadModuleMapFile(StringRef ModuleMapPath) override { - Out.indent(2) << "Module map file: " << ModuleMapPath << "\n"; - } - - bool ReadLanguageOptions(const LangOptions &LangOpts, - StringRef ModuleFilename, bool /*Complain*/, - bool /*AllowCompatibleDifferences*/) override { - Out.indent(2) << "Language options:\n"; -#define LANGOPT(Name, Bits, Default, Description) \ - DUMP_BOOLEAN(LangOpts.Name, Description); -#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \ - Out.indent(4) << Description << ": " \ - << static_cast(LangOpts.get##Name()) << "\n"; -#define VALUE_LANGOPT(Name, Bits, Default, Description) \ - Out.indent(4) << Description << ": " << LangOpts.Name << "\n"; -#define BENIGN_LANGOPT(Name, Bits, Default, Description) -#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description) -#include "clang/Basic/LangOptions.def" - - if (!LangOpts.ModuleFeatures.empty()) { - Out.indent(4) << "Module features:\n"; - for (StringRef Feature : LangOpts.ModuleFeatures) - Out.indent(6) << Feature << "\n"; - } - - return false; - } - - bool ReadTargetOptions(const TargetOptions &TargetOpts, - StringRef ModuleFilename, - bool /*Complain*/, - bool /*AllowCompatibleDifferences*/) override { - Out.indent(2) << "Target options:\n"; - Out.indent(4) << " Triple: " << TargetOpts.Triple << "\n"; - Out.indent(4) << " CPU: " << TargetOpts.CPU << "\n"; - Out.indent(4) << " ABI: " << TargetOpts.ABI << "\n"; - - if (!TargetOpts.FeaturesAsWritten.empty()) { - Out.indent(4) << "Target features:\n"; - for (unsigned I = 0, N = TargetOpts.FeaturesAsWritten.size(); - I != N; ++I) { - Out.indent(6) << TargetOpts.FeaturesAsWritten[I] << "\n"; - } - } - - return false; - } - - bool ReadDiagnosticOptions(IntrusiveRefCntPtr DiagOpts, - StringRef ModuleFilename, - bool /*Complain*/) override { - Out.indent(2) << "Diagnostic options:\n"; -#define DIAGOPT(Name, Bits, Default) DUMP_BOOLEAN(DiagOpts->Name, #Name); -#define ENUM_DIAGOPT(Name, Type, Bits, Default) \ - Out.indent(4) << #Name << ": " << DiagOpts->get##Name() << "\n"; -#define VALUE_DIAGOPT(Name, Bits, Default) \ - Out.indent(4) << #Name << ": " << DiagOpts->Name << "\n"; -#include "clang/Basic/DiagnosticOptions.def" - - Out.indent(4) << "Diagnostic flags:\n"; - for (const std::string &Warning : DiagOpts->Warnings) - Out.indent(6) << "-W" << Warning << "\n"; - for (const std::string &Remark : DiagOpts->Remarks) - Out.indent(6) << "-R" << Remark << "\n"; - - return false; - } - - bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, - StringRef ModuleFilename, - StringRef SpecificModuleCachePath, - bool /*Complain*/) override { - Out.indent(2) << "Header search options:\n"; - Out.indent(4) << "System root [-isysroot=]: '" - << HSOpts.Sysroot << "'\n"; - Out.indent(4) << "Resource dir [ -resource-dir=]: '" - << HSOpts.ResourceDir << "'\n"; - Out.indent(4) << "Module Cache: '" << SpecificModuleCachePath - << "'\n"; - DUMP_BOOLEAN(HSOpts.UseBuiltinIncludes, - "Use builtin include directories [-nobuiltininc]"); - DUMP_BOOLEAN(HSOpts.UseStandardSystemIncludes, - "Use standard system include directories [-nostdinc]"); - DUMP_BOOLEAN(HSOpts.UseStandardCXXIncludes, - "Use standard C++ include directories [-nostdinc++]"); - DUMP_BOOLEAN(HSOpts.UseLibcxx, - "Use libc++ (rather than libstdc++) [-stdlib=]"); - return false; - } - - bool - ReadPreprocessorOptions(const PreprocessorOptions& PPOpts, - StringRef /*ModuleFilename*/, - bool /*ReadMacros*/, bool /*Complain*/, - std::string& /*SuggestedPredefines*/) override { - Out.indent(2) << "Preprocessor options:\n"; - DUMP_BOOLEAN(PPOpts.UsePredefines, - "Uses compiler/target-specific predefines [-undef]"); - DUMP_BOOLEAN(PPOpts.DetailedRecord, - "Uses detailed preprocessing record (for indexing)"); - - if (!PPOpts.Macros.empty()) { - Out.indent(4) << "Predefined macros:\n"; - } - - for (std::vector >::const_iterator - I = PPOpts.Macros.begin(), IEnd = PPOpts.Macros.end(); - I != IEnd; ++I) { - Out.indent(6); - if (I->second) - Out << "-U"; - else - Out << "-D"; - Out << I->first << "\n"; - } - return false; - } - - /// Indicates that a particular module file extension has been read. - void readModuleFileExtension( - const ModuleFileExtensionMetadata &Metadata) override { - Out.indent(2) << "Module file extension '" - << Metadata.BlockName << "' " << Metadata.MajorVersion - << "." << Metadata.MinorVersion; - if (!Metadata.UserInfo.empty()) { - Out << ": "; - Out.write_escaped(Metadata.UserInfo); - } - - Out << "\n"; - } - - /// Tells the \c ASTReaderListener that we want to receive the - /// input files of the AST file via \c visitInputFile. - bool needsInputFileVisitation() override { return true; } - - /// Tells the \c ASTReaderListener that we want to receive the - /// input files of the AST file via \c visitInputFile. - bool needsSystemInputFileVisitation() override { return true; } - - /// Indicates that the AST file contains particular input file. - /// - /// \returns true to continue receiving the next input file, false to stop. - bool visitInputFile(StringRef Filename, bool isSystem, - bool isOverridden, bool isExplicitModule) override { - - Out.indent(2) << "Input file: " << Filename; - - if (isSystem || isOverridden || isExplicitModule) { - Out << " ["; - if (isSystem) { - Out << "System"; - if (isOverridden || isExplicitModule) - Out << ", "; - } - if (isOverridden) { - Out << "Overridden"; - if (isExplicitModule) - Out << ", "; - } - if (isExplicitModule) - Out << "ExplicitModule"; - - Out << "]"; - } - - Out << "\n"; - - return true; - } -#undef DUMP_BOOLEAN - }; - - std::unique_ptr OutFile; - StringRef OutputFileName = FrontendOpts.OutputFile; - if (!OutputFileName.empty() && OutputFileName != "-") { - std::error_code EC; - OutFile.reset(new llvm::raw_fd_ostream(OutputFileName.str(), EC, - llvm::sys::fs::OF_Text)); - } - llvm::raw_ostream &Out = OutFile.get()? *OutFile.get() : llvm::outs(); - StringRef CurInput = FrontendOpts.Inputs[0].getFile(); - Out << "Information for module file '" << CurInput << "':\n"; - auto &FileMgr = CI.getFileManager(); - auto Buffer = FileMgr.getBufferForFile(CurInput); - StringRef Magic = (*Buffer)->getMemBufferRef().getBuffer(); - bool IsRaw = (Magic.size() >= 4 && Magic[0] == 'C' && Magic[1] == 'P' && - Magic[2] == 'C' && Magic[3] == 'H'); - Out << " Module format: " << (IsRaw ? "raw" : "obj") << "\n"; - Preprocessor &PP = CI.getPreprocessor(); - DumpModuleInfoListener Listener(Out); - HeaderSearchOptions &HSOpts = - PP.getHeaderSearchInfo().getHeaderSearchOpts(); - ASTReader::readASTFileControlBlock(CurInput, FileMgr, CI.getModuleCache(), - CI.getPCHContainerReader(), - /*FindModuleFileExtensions=*/true, - Listener, - HSOpts.ModulesValidateDiagnosticOptions); - } - } - static CompilerInstance* createCIImpl(std::unique_ptr Buffer, const CompilerOptions& COpts, const char* LLVMDir, @@ -1516,8 +1296,8 @@ namespace { // Copied from CompilerInstance::createDiagnostics: // Chain in -verify checker, if requested. - if (DiagOpts.VerifyDiagnostics) - Diags->setClient(new clang::VerifyDiagnosticConsumer(*Diags)); + // if (DiagOpts.VerifyDiagnostics) + // Diags->setClient(new clang::VerifyDiagnosticConsumer(*Diags)); IntrusiveRefCntPtr Overlay = new llvm::vfs::OverlayFileSystem(llvm::vfs::getRealFileSystem()); @@ -1634,8 +1414,8 @@ namespace { /*UserFilesAreVolatile*/ true); CI->setSourceManager(SM); // CI now owns SM - if (CI->getCodeGenOpts().TimePasses) - CI->createFrontendTimer(); + // if (CI->getCodeGenOpts().TimePasses) + // CI->createFrontendTimer(); if (FrontendOpts.ModulesEmbedAllFiles) CI->getSourceManager().setAllFilesAreTransient(true); @@ -1653,110 +1433,144 @@ namespace { // Build the virtual file, Give it a name that's likely not to ever // be #included (so we won't get a clash in clang's cache). - const char* Filename = "<<< cling interactive line includer >>>"; - FileEntryRef FE = FM.getVirtualFileRef(Filename, 1U << 15U, time(0)); + // const char* Filename = "<<< cling interactive line includer >>>"; + // FileEntryRef FE = FM.getVirtualFileRef(Filename, 1U << 15U, time(0)); - // Tell ASTReader to create a FileID even if this file does not exist: - SM->setFileIsTransient(FE); - FileID MainFileID = SM->createFileID(FE, SourceLocation(), SrcMgr::C_User); - SM->setMainFileID(MainFileID); + // // Tell ASTReader to create a FileID even if this file does not exist: + // SM->setFileIsTransient(FE); + // FileID MainFileID = SM->createFileID(FE, SourceLocation(), SrcMgr::C_User); + // SM->setMainFileID(MainFileID); if (!Buffer) Buffer = llvm::MemoryBuffer::getMemBuffer("/*CLING DEFAULT MEMBUF*/;\n"); + // SM->overrideFileContents(FE, std::move(Buffer)); // Adapted from upstream clang/lib/Interpreter/Interpreter.cpp // FIXME: Merge with CompilerInstance::ExecuteAction. llvm::MemoryBuffer* MB = Buffer.release(); - CI->getPreprocessorOpts().addRemappedFile(Filename, MB); + if (MB) { + CI->getFrontendOpts().Inputs.clear(); + // CI->getFrontendOpts().Inputs.emplace_back(Filename, + // InputKind(Language::CXX));//, true, + // // 1U << 15U, + // // MB->getMemBufferRef()); + CI->getFrontendOpts().Inputs.emplace_back(MB->getMemBufferRef(), + InputKind(Language::CXX));//, true, + // 1U << 15U, + // MB->getMemBufferRef()); + } + CI->setIncrementalSourceMgrInitializer([&](CompilerInstance& CI, const FrontendInputFile &Input) { + const char* Filename = "<<< cling interactive line includer >>>"; + auto& FM = CI.getFileManager(); + auto& SM = CI.getSourceManager(); + FileEntryRef FE = FM.getVirtualFileRef(Filename, 1U << 15U, time(0)); + + // Tell ASTReader to create a FileID even if this file does not exist: + SM.setFileIsTransient(FE); + FileID MainFileID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); + SM.setMainFileID(MainFileID); + + std::unique_ptr Buffer = nullptr; + if (Input.isBuffer()) + Buffer = llvm::MemoryBuffer::getMemBuffer(Input.getBuffer()); + else + Buffer = llvm::MemoryBuffer::getMemBuffer(""); + SM.overrideFileContents(FE, std::move(Buffer)); + return true; + }); + + // CI->getFrontendOpts().Inputs.clear(); + // CI->getFrontendOpts().Inputs.emplace_back(Filename, InputKind(Language::CXX)); + // CI->getPreprocessorOpts().addRemappedFile(Filename, MB); // Create TargetInfo for the other side of CUDA and OpenMP compilation. - if ((CI->getLangOpts().CUDA || CI->getLangOpts().OpenMPIsTargetDevice) && - !CI->getFrontendOpts().AuxTriple.empty()) { - auto TO = std::make_shared(); - TO->Triple = CI->getFrontendOpts().AuxTriple; - TO->HostTriple = CI->getTarget().getTriple().str(); - CI->setAuxTarget(TargetInfo::CreateTargetInfo(CI->getDiagnostics(), TO)); - } + // if ((CI->getLangOpts().CUDA || CI->getLangOpts().OpenMPIsTargetDevice) && + // !CI->getFrontendOpts().AuxTriple.empty()) { + // auto TO = std::make_shared(); + // TO->Triple = CI->getFrontendOpts().AuxTriple; + // TO->HostTriple = CI->getTarget().getTriple().str(); + // CI->setAuxTarget(TargetInfo::CreateTargetInfo(CI->getDiagnostics(), TO)); + // } // Set up the preprocessor - auto TUKind = COpts.ModuleName.empty() ? TU_Complete : TU_ClangModule; - CI->createPreprocessor(TUKind); + // auto TUKind = COpts.ModuleName.empty() ? TU_Complete : TU_ClangModule; + // CI->createPreprocessor(TUKind); // With modules, we now start adding prebuilt module paths to the CI. // Modules from those paths are treated like they are never out of date // and we don't update them on demand. // This mostly helps ROOT where we can't just recompile any out of date // modules because we would miss the annotations that rootcling creates. - if (COpts.CxxModules) { - setupCxxModules(*CI); - } + // if (COpts.CxxModules) { + // setupCxxModules(*CI); + // } - Preprocessor& PP = CI->getPreprocessor(); + // Preprocessor& PP = CI->getPreprocessor(); - PP.getBuiltinInfo().initializeBuiltins(PP.getIdentifierTable(), - PP.getLangOpts()); + // PP.getBuiltinInfo().initializeBuiltins(PP.getIdentifierTable(), + // PP.getLangOpts()); - // Set up the ASTContext - CI->createASTContext(); + // // Set up the ASTContext + // CI->createASTContext(); - std::vector> Consumers; + // std::vector> Consumers; - if (!OnlyLex && !AutoComplete) { - assert(customConsumer && "Need to specify a custom consumer" - " when not in OnlyLex mode"); - Consumers.push_back(std::move(customConsumer)); - } + // if (!OnlyLex && !AutoComplete) { + // assert(customConsumer && "Need to specify a custom consumer" + // " when not in OnlyLex mode"); + // Consumers.push_back(std::move(customConsumer)); + // } // With C++ modules, we now attach the consumers that will handle the // generation of the PCM file itself in case we want to generate // a C++ module with the current interpreter instance. - if (COpts.CxxModules && !COpts.ModuleName.empty()) { - // Code below from the (private) code in the GenerateModuleAction class. - llvm::SmallVector Output; - llvm::sys::path::append(Output, COpts.CachePath, - COpts.ModuleName + ".pcm"); - StringRef ModuleOutputFile = StringRef(Output.data(), Output.size()); - - std::unique_ptr OS = - CI->createOutputFile(ModuleOutputFile, /*Binary=*/true, - /*RemoveFileOnSignal=*/false, - /*useTemporary=*/true, - /*CreateMissingDirectories=*/true); - assert(OS); - - std::string Sysroot; - - auto PCHBuff = std::make_shared(); - - Consumers.push_back(std::make_unique( - CI->getPreprocessor(), CI->getModuleCache(), ModuleOutputFile, - Sysroot, PCHBuff, CI->getFrontendOpts().ModuleFileExtensions, - /*AllowASTWithErrors=*/false, - /*IncludeTimestamps=*/ - +CI->getFrontendOpts().BuildingImplicitModule)); - Consumers.push_back( - CI->getPCHContainerWriter().CreatePCHContainerGenerator( - *CI, "", ModuleOutputFile.str(), std::move(OS), PCHBuff)); - - // Set the current module name for clang. With that clang doesn't start - // to build the current module on demand when we include a header - // from the current module. - CI->getLangOpts().CurrentModule = COpts.ModuleName; - CI->getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); - - // Push the current module to the build stack so that clang knows when - // we have a cyclic dependency. - SM->pushModuleBuildStack(COpts.ModuleName, - FullSourceLoc(SourceLocation(), *SM)); - } - - std::unique_ptr multiConsumer( - new clang::MultiplexConsumer(std::move(Consumers))); - CI->setASTConsumer(std::move(multiConsumer)); + // if (COpts.CxxModules && !COpts.ModuleName.empty()) { + // // Code below from the (private) code in the GenerateModuleAction class. + // llvm::SmallVector Output; + // llvm::sys::path::append(Output, COpts.CachePath, + // COpts.ModuleName + ".pcm"); + // StringRef ModuleOutputFile = StringRef(Output.data(), Output.size()); + + // std::unique_ptr OS = + // CI->createOutputFile(ModuleOutputFile, /*Binary=*/true, + // /*RemoveFileOnSignal=*/false, + // /*useTemporary=*/true, + // /*CreateMissingDirectories=*/true); + // assert(OS); + + // std::string Sysroot; + + // auto PCHBuff = std::make_shared(); + + // Consumers.push_back(std::make_unique( + // CI->getPreprocessor(), CI->getModuleCache(), ModuleOutputFile, + // Sysroot, PCHBuff, CI->getFrontendOpts().ModuleFileExtensions, + // /*AllowASTWithErrors=*/false, + // /*IncludeTimestamps=*/ + // +CI->getFrontendOpts().BuildingImplicitModule)); + // Consumers.push_back( + // CI->getPCHContainerWriter().CreatePCHContainerGenerator( + // *CI, "", ModuleOutputFile.str(), std::move(OS), PCHBuff)); + + // // Set the current module name for clang. With that clang doesn't start + // // to build the current module on demand when we include a header + // // from the current module. + // CI->getLangOpts().CurrentModule = COpts.ModuleName; + // CI->getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); + + // // Push the current module to the build stack so that clang knows when + // // we have a cyclic dependency. + // SM->pushModuleBuildStack(COpts.ModuleName, + // FullSourceLoc(SourceLocation(), *SM)); + // } + + // std::unique_ptr multiConsumer( + // new clang::MultiplexConsumer(std::move(Consumers))); + // CI->setASTConsumer(std::move(multiConsumer)); // Set up Sema - CodeCompleteConsumer* CCC = 0; + // CodeCompleteConsumer* CCC = 0; // Make sure we inform Sema we compile a Module. - CI->createSema(TUKind, CCC); + // CI->createSema(TUKind, CCC); // Set CodeGen options. CodeGenOptions& CGOpts = CI->getCodeGenOpts(); @@ -1786,6 +1600,8 @@ namespace { // aliasing the complete ctor to the base ctor causes the JIT to crash CGOpts.CXXCtorDtorAliases = 0; CGOpts.VerifyModule = 0; // takes too long + CGOpts.ClearASTBeforeBackend = false; + CGOpts.DisableFree = false; if (!OnlyLex) { // -nobuiltininc @@ -1804,28 +1620,28 @@ namespace { // 2. It could corrupt clang's directory cache // HeaderSearchOptions.::AddSearchPath is a better alternative - clang::ApplyHeaderSearchOptions(PP.getHeaderSearchInfo(), HOpts, - PP.getLangOpts(), - PP.getTargetInfo().getTriple()); + // clang::ApplyHeaderSearchOptions(PP.getHeaderSearchInfo(), HOpts, + // PP.getLangOpts(), + // PP.getTargetInfo().getTriple()); } // Tell the diagnostic client that we are entering file parsing mode as the // handling of modulemap files may issue diagnostics. // FIXME: Consider moving in SetupDiagnostics. - DiagnosticConsumer& DClient = CI->getDiagnosticClient(); - DClient.BeginSourceFile(CI->getLangOpts(), &PP); - - for (const auto& ModuleMapFile : FrontendOpts.ModuleMapFiles) { - auto File = FM.getFileRef(ModuleMapFile); - if (!File) { - CI->getDiagnostics().Report(diag::err_module_map_not_found) - << ModuleMapFile; - continue; - } - PP.getHeaderSearchInfo().loadModuleMapFile(*File, /*IsSystem*/ false); - } + // DiagnosticConsumer& DClient = CI->getDiagnosticClient(); + // DClient.BeginSourceFile(CI->getLangOpts(), &PP); - HandleProgramActions(*CI); + // for (const auto& ModuleMapFile : FrontendOpts.ModuleMapFiles) { + // auto File = FM.getFileRef(ModuleMapFile); + // if (!File) { + // CI->getDiagnostics().Report(diag::err_module_map_not_found) + // << ModuleMapFile; + // continue; + // } + // PP.getHeaderSearchInfo().loadModuleMapFile(*File, /*IsSystem*/ false); + // } + + // HandleProgramActions(*CI); return CI.release(); // Passes over the ownership to the caller. } @@ -1834,6 +1650,10 @@ namespace { namespace cling { + void CIFactory::collectModule(CompilerInstance &CI) { + setupCxxModules(CI); + } + CompilerInstance* CIFactory::createCI( llvm::StringRef Code, const InvocationOptions& Opts, const char* LLVMDir, std::optional> consumerOpt, @@ -1855,6 +1675,8 @@ namespace cling { consumerOpt ? std::move(*consumerOpt) : nullptr, moduleExtensions, OnlyLex); } - + unsigned CIFactory::CxxStdCompiledWith() { + return ::CxxStdCompiledWith(); + } } // namespace cling diff --git a/interpreter/cling/lib/Interpreter/DeclCollector.cpp b/interpreter/cling/lib/Interpreter/DeclCollector.cpp index 8cf0f0c7861f3..418014fd60098 100644 --- a/interpreter/cling/lib/Interpreter/DeclCollector.cpp +++ b/interpreter/cling/lib/Interpreter/DeclCollector.cpp @@ -103,10 +103,10 @@ namespace cling { } }; - void DeclCollector::Setup(IncrementalParser* IncrParser, + void DeclCollector::Setup(//IncrementalParser* IncrParser, std::unique_ptr Consumer, clang::Preprocessor& PP) { - m_IncrParser = IncrParser; + // m_IncrParser = IncrParser; m_Consumer = std::move(Consumer); PP.addPPCallbacks(std::unique_ptr(new PPAdapter(this))); } diff --git a/interpreter/cling/lib/Interpreter/DeclCollector.h b/interpreter/cling/lib/Interpreter/DeclCollector.h index e96f3ce80e0cf..2638c181587bd 100644 --- a/interpreter/cling/lib/Interpreter/DeclCollector.h +++ b/interpreter/cling/lib/Interpreter/DeclCollector.h @@ -52,7 +52,7 @@ namespace cling { /// std::vector> m_WrapperTransformers; - IncrementalParser* m_IncrParser = nullptr; + // IncrementalParser* m_IncrParser = nullptr; std::unique_ptr m_Consumer; Transaction* m_CurTransaction = nullptr; @@ -86,7 +86,7 @@ namespace cling { WT->SetConsumer(this); } - void Setup(IncrementalParser* IncrParser, + void Setup(//IncrementalParser* IncrParser, std::unique_ptr Consumer, clang::Preprocessor& PP); diff --git a/interpreter/cling/lib/Interpreter/IncrementalAction.h b/interpreter/cling/lib/Interpreter/IncrementalAction.h new file mode 100644 index 0000000000000..ee065a588779b --- /dev/null +++ b/interpreter/cling/lib/Interpreter/IncrementalAction.h @@ -0,0 +1,96 @@ +//--------------------------------------------------------------------*- C++ -*- +// CLING - the C++ LLVM-based InterpreterG :) +// author: Axel Naumann +// +// This file is dual-licensed: you can choose to license it under the University +// of Illinois Open Source License or the GNU Lesser General Public License. See +// LICENSE.TXT for details. +//------------------------------------------------------------------------------ + +#ifndef CLING_INCREMENTAL_ACTION_H +#define CLING_INCREMENTAL_ACTION_H + +#include "cling/Interpreter/InvocationOptions.h" + +#include "clang/Frontend/FrontendActions.h" +#include "clang/Frontend/MultiplexConsumer.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include +#include + +namespace llvm { + class Module; +} + +namespace clang { + class ASTConsumer; + class CodeGenerator; + class CompilerInstance; +} // namespace clang + +namespace cling { + class DeclCollector; + class Interpreter; + + class IncrementalAction : public clang::WrapperFrontendAction { + private: + bool IsTerminating = false; + clang::CompilerInstance& CI; + CompilerOptions COpts; + cling::DeclCollector* DeclCollectorConsumer; + + /// When CodeGen is created the first llvm::Module gets cached in many + /// places and we must keep it alive. + std::unique_ptr CachedInCodeGenModule; + + public: + IncrementalAction(clang::CompilerInstance& CI, llvm::LLVMContext& LLVMCtx, + CompilerOptions COpts, llvm::Error& Err); + + clang::FrontendAction* getWrapped() const { return WrappedAction.get(); } + + clang::TranslationUnitKind getTranslationUnitKind() override { + return clang::TU_Incremental; + } + + DeclCollector* getDeclCollectorConsumer() const { + assert(DeclCollectorConsumer); + return DeclCollectorConsumer; + } + + std::vector> + CreateMultiplexConsumer(clang::CompilerInstance& CI, + llvm::StringRef InFile); + + std::unique_ptr + CreateASTConsumer(clang::CompilerInstance& CI, + llvm::StringRef InFile) override; + + void ExecuteAction() override; + + bool BeginSourceFileAction(clang::CompilerInstance& CI) override; + + void EndSourceFile() override { + if (IsTerminating && getWrapped()) + clang::WrapperFrontendAction::EndSourceFile(); + } + + void FinalizeAction() { + assert(!IsTerminating && "Already finalized!"); + IsTerminating = true; + EndSourceFile(); + } + + std::unique_ptr GenModule(); + + clang::CodeGenerator* getCodeGen(); + + void CacheCodeGenModule(); + + llvm::Module* getCachedCodeGenModule() const; + }; +} // end namespace cling +#endif // CLING_INCREMENTAL_ACTION_H \ No newline at end of file diff --git a/interpreter/cling/lib/Interpreter/IncrementalParser.cpp b/interpreter/cling/lib/Interpreter/IncrementalParser.cpp index 718fa13a73208..ee1307ecba55d 100644 --- a/interpreter/cling/lib/Interpreter/IncrementalParser.cpp +++ b/interpreter/cling/lib/Interpreter/IncrementalParser.cpp @@ -18,6 +18,7 @@ #include "DefinitionShadower.h" #include "DeviceKernelInliner.h" #include "DynamicLookup.h" +#include "IncrementalAction.h" #include "NullDerefProtectionTransformer.h" #include "TransactionPool.h" #include "ValueExtractionSynthesizer.h" @@ -36,13 +37,18 @@ #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/FileManager.h" +#include "clang/CodeGen/CodeGenAction.h" #include "clang/CodeGen/ModuleBuilder.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/FrontendOptions.h" #include "clang/Frontend/FrontendPluginRegistry.h" #include "clang/Frontend/MultiplexConsumer.h" +#include "clang/FrontendTool/Utils.h" #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" +#include "clang/Parse/ParseAST.h" #include "clang/Parse/Parser.h" #include "clang/Sema/Sema.h" #include "clang/Sema/SemaDiagnostic.h" @@ -235,109 +241,206 @@ namespace { }; } // unnamed namespace -static void HandlePlugins(CompilerInstance& CI, - std::vector>& Consumers) { - // Copied from Frontend/FrontendAction.cpp. - // FIXME: Remove when we switch to a tools-based cling driver. - - // If the FrontendPluginRegistry has plugins before loading any shared library - // this means we have linked our plugins. This is useful when cling runs in - // embedded mode (in a shared library). This is the only feasible way to have - // plugins if cling is in a single shared library which is dlopen-ed with - // RTLD_LOCAL. In that situation plugins can still find the cling, clang and - // llvm symbols opened with local visibility. - if (FrontendPluginRegistry::begin() == FrontendPluginRegistry::end()) { - for (const std::string& Path : CI.getFrontendOpts().Plugins) { - std::string Err; - if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(Path.c_str(), &Err)) - CI.getDiagnostics().Report(clang::diag::err_fe_unable_to_load_plugin) - << Path << Err; +namespace cling { + + IncrementalAction::IncrementalAction(CompilerInstance& CI, + llvm::LLVMContext& LLVMCtx, + // Interpreter& m_Interp, + CompilerOptions COpts, llvm::Error& Err) + : WrapperFrontendAction([&]() { + llvm::ErrorAsOutParameter EAO(&Err); + std::unique_ptr Act; + switch (CI.getFrontendOpts().ProgramAction) { + default: + Err = llvm::createStringError( + std::errc::state_not_recoverable, + "Driver initialization failed. " + "Incremental mode for action %d is not supported", + CI.getFrontendOpts().ProgramAction); + return Act; + case frontend::ASTDump: + case frontend::ASTPrint: + case frontend::ParseSyntaxOnly: + Act = CreateFrontendAction(CI); + break; + case frontend::PluginAction: + case frontend::EmitAssembly: + case frontend::EmitBC: + case frontend::EmitObj: + case frontend::PrintPreprocessedInput: + case frontend::EmitLLVMOnly: + Act.reset(new EmitLLVMOnlyAction(&LLVMCtx)); + break; + } + return Act; + }()), + CI(CI), // IncrParser(IncrParser), + // m_Interpreter(m_Interp), + COpts(COpts) {} + + std::vector> + IncrementalAction::CreateMultiplexConsumer(CompilerInstance& CI, + StringRef InFile) { + std::vector> Consumers; + // With C++ modules, we now attach the consumers that will handle the + // generation of the PCM file itself in case we want to generate + // a C++ module with the current interpreter instance. + if (COpts.CxxModules && !COpts.ModuleName.empty()) { + // Code below from the (private) code in the GenerateModuleAction class. + llvm::SmallVector Output; + llvm::sys::path::append(Output, COpts.CachePath, + COpts.ModuleName + ".pcm"); + StringRef ModuleOutputFile = StringRef(Output.data(), Output.size()); + + std::unique_ptr OS = + CI.createOutputFile(ModuleOutputFile, /*Binary=*/true, + /*RemoveFileOnSignal=*/false, + /*useTemporary=*/true, + /*CreateMissingDirectories=*/true); + assert(OS); + + std::string Sysroot; + + auto PCHBuff = std::make_shared(); + + Consumers.push_back(std::make_unique( + CI.getPreprocessor(), CI.getModuleCache(), ModuleOutputFile, Sysroot, + PCHBuff, CI.getFrontendOpts().ModuleFileExtensions, + /*AllowASTWithErrors=*/false, + /*IncludeTimestamps=*/ + +CI.getFrontendOpts().BuildingImplicitModule)); + Consumers.push_back( + CI.getPCHContainerWriter().CreatePCHContainerGenerator( + CI, "", ModuleOutputFile.str(), std::move(OS), PCHBuff)); + } + + return Consumers; + } + + std::unique_ptr + IncrementalAction::CreateASTConsumer(CompilerInstance& CI, + StringRef InFile) { + auto C = WrapperFrontendAction::CreateASTConsumer(CI, InFile); + auto DC = std::make_unique(); + DeclCollectorConsumer = DC.get(); + DC->Setup(std::move(C), CI.getPreprocessor()); + std::vector> Cs = + CreateMultiplexConsumer(CI, InFile); + if (!Cs.empty()) { + Cs.insert(Cs.begin(), std::move(DC)); + // Cs.push_back(std::move(DC)); + return std::make_unique(std::move(Cs)); } - // If we are not statically linked, we should register the pragmas ourselves - // because the dlopen happens after creating the clang::Preprocessor which - // calls RegisterBuiltinPragmas. - // FIXME: This can be avoided by refactoring our routine and moving it to - // the CIFactory. This requires an abstraction which allows us to - // conditionally create MultiplexingConsumers. - - // Copied from Lex/Pragma.cpp - // Pragmas added by plugins - for (PragmaHandlerRegistry::iterator it = PragmaHandlerRegistry::begin(), - ie = PragmaHandlerRegistry::end(); it != ie; ++it) - CI.getPreprocessor().AddPragmaHandler(it->instantiate().release()); + return std::move(DC); } - for (auto it = clang::FrontendPluginRegistry::begin(), - ie = clang::FrontendPluginRegistry::end(); - it != ie; ++it) { - std::unique_ptr P(it->instantiate()); + void IncrementalAction::ExecuteAction() { + // Interpreter::PushTransactionRAII PushedT(&m_Interpreter); + // WrapperFrontendAction::ExecuteAction(); + // getCompilerInstance().getSema().CurContext = nullptr; + CompilerInstance& CI = getCompilerInstance(); + if (!CI.hasPreprocessor()) + return; - PluginASTAction::ActionType PluginActionType = P->getActionType(); - assert(PluginActionType != clang::PluginASTAction::ReplaceAction); + // FIXME: Move the truncation aspect of this into Sema, we delayed this + // till here so the source manager would be initialized. + if (hasCodeCompletionSupport() && + !CI.getFrontendOpts().CodeCompletionAt.FileName.empty()) + CI.createCodeCompletionConsumer(); - if (P->ParseArgs(CI, CI.getFrontendOpts().PluginArgs[it->getName().str()])) { - std::unique_ptr PluginConsumer - = P->CreateASTConsumer(CI, /*InputFile*/ ""); - if (PluginActionType == clang::PluginASTAction::AddBeforeMainAction) - Consumers.insert(Consumers.begin(), std::move(PluginConsumer)); - else - Consumers.push_back(std::move(PluginConsumer)); + // Use a code completion consumer? + CodeCompleteConsumer* CompletionConsumer = nullptr; + if (CI.hasCodeCompletionConsumer()) + CompletionConsumer = &CI.getCodeCompletionConsumer(); + + if (!CI.hasSema()) + CI.createSema(getTranslationUnitKind(), CompletionConsumer); + + // Interpreter::PushTransactionRAII PushedT(&m_Interpreter); + // ParseAST(CI.getSema(), CI.getFrontendOpts().ShowStats, + // CI.getFrontendOpts().SkipFunctionBodies); + // getCompilerInstance().getSema().CurContext = nullptr; + } + + bool IncrementalAction::BeginSourceFileAction(CompilerInstance& CI) { + if (COpts.CxxModules) + CIFactory::collectModule(CI); + + if (COpts.CxxModules && !COpts.ModuleName.empty()) { + // Set the current module name for clang. With that clang doesn't start + // to build the current module on demand when we include a header + // from the current module. + CI.getLangOpts().CurrentModule = COpts.ModuleName; + CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); + + SourceManager& SM = CI.getSourceManager(); + // Push the current module to the build stack so that clang knows when + // we have a cyclic dependency. + + SM.pushModuleBuildStack(COpts.ModuleName, + FullSourceLoc(SourceLocation(), SM)); } + + return WrapperFrontendAction::BeginSourceFileAction(CI); } -} -namespace cling { - IncrementalParser::IncrementalParser(Interpreter* interp, const char* llvmdir, - const ModuleFileExtensions& moduleExtensions) - : m_Interpreter(interp) { - std::unique_ptr consumer; - consumer.reset(m_Consumer = new cling::DeclCollector()); - m_CI.reset(CIFactory::createCI("\n", interp->getOptions(), llvmdir, - std::make_optional(std::move(consumer)), - moduleExtensions)); - - if (!m_CI) { - cling::errs() << "Compiler instance could not be created.\n"; - return; + std::unique_ptr IncrementalAction::GenModule() { + static unsigned ID = 0; + if (CodeGenerator* CG = getCodeGen()) { + // Clang's CodeGen is designed to work with a single llvm::Module. In + // many cases for convenience various CodeGen parts have a reference to + // the llvm::Module (TheModule or Module) which does not change when a + // new module is pushed. However, the execution engine wants to take + // ownership of the module which does not map well to CodeGen's design. + // To work this around we created an empty module to make CodeGen happy. + // We should make sure it always stays empty. + assert(((!CachedInCodeGenModule || + !CI.getPreprocessorOpts().Includes.empty() || + !CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) || + (CachedInCodeGenModule->empty() && + CachedInCodeGenModule->global_empty() && + CachedInCodeGenModule->alias_empty() && + CachedInCodeGenModule->ifunc_empty())) && + "CodeGen wrote to a readonly module"); + std::unique_ptr M(CG->ReleaseModule()); + CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext()); + return M; } - // Is the CompilerInstance being used to generate output only? - if (m_Interpreter->getOptions().CompilerOpts.HasOutput) - return; + return nullptr; + } + CodeGenerator* IncrementalAction::getCodeGen() { + FrontendAction* WrappedAct = getWrapped(); + return static_cast(WrappedAct)->getCodeGenerator(); + } + + void IncrementalAction::CacheCodeGenModule() { CachedInCodeGenModule = GenModule(); } + + llvm::Module* IncrementalAction::getCachedCodeGenModule() const { + return CachedInCodeGenModule.get(); + } + + IncrementalParser::IncrementalParser(Interpreter* interp, CompilerInstance* CI, + IncrementalAction* Act) + : m_Interpreter(interp), m_CI(CI), m_Act(Act) { + m_Consumer = m_Act->getDeclCollectorConsumer(); if (!m_Consumer) { cling::errs() << "No AST consumer available.\n"; return; } - - std::vector> Consumers; - HandlePlugins(*m_CI, Consumers); - std::unique_ptr WrappedConsumer; - - DiagnosticsEngine& Diag = m_CI->getDiagnostics(); if (m_CI->getFrontendOpts().ProgramAction != frontend::ParseSyntaxOnly) { - auto CG - = std::unique_ptr(CreateLLVMCodeGen(Diag, - makeModuleName(), - &m_CI->getVirtualFileSystem(), - m_CI->getHeaderSearchOpts(), - m_CI->getPreprocessorOpts(), - m_CI->getCodeGenOpts(), - *m_Interpreter->getLLVMContext()) - ); - m_CodeGen = CG.get(); + if (m_Act->getCodeGen()) + m_CodeGen = m_Act->getCodeGen(); + assert(m_CodeGen); - if (!Consumers.empty()) { - Consumers.push_back(std::move(CG)); - WrappedConsumer.reset(new MultiplexConsumer(std::move(Consumers))); - } - else - WrappedConsumer = std::move(CG); } - // Initialize the DeclCollector and add callbacks keeping track of macros. - m_Consumer->Setup(this, std::move(WrappedConsumer), m_CI->getPreprocessor()); + // Is the CompilerInstance being used to generate output only? + if (m_Interpreter->getOptions().CompilerOpts.HasOutput) + return; + DiagnosticsEngine& Diag = m_CI->getDiagnostics(); m_DiagConsumer.reset(new FilteringDiagConsumer(Diag, false)); initializeVirtualFile(); @@ -347,31 +450,35 @@ namespace cling { IncrementalParser::Initialize(llvm::SmallVectorImpl& result, bool isChildInterpreter) { m_TransactionPool.reset(new TransactionPool); + + if (m_CI->getPreprocessor().TUKind != TU_Incremental) + return true; // This a one-time, non-incremental action. + if (hasCodeGenerator()) getCodeGenerator()->Initialize(getCI()->getASTContext()); + Preprocessor& PP = m_CI->getPreprocessor(); CompilationOptions CO = m_Interpreter->makeDefaultCompilationOpts(); Transaction* CurT = beginTransaction(CO); - Preprocessor& PP = m_CI->getPreprocessor(); - DiagnosticsEngine& Diags = m_CI->getSema().getDiagnostics(); + // DiagnosticsEngine& Diags = m_CI->getSema().getDiagnostics(); // Pull in PCH. - const std::string& PCHFileName - = m_CI->getInvocation().getPreprocessorOpts().ImplicitPCHInclude; - if (!PCHFileName.empty()) { - Transaction* PchT = beginTransaction(CO); - DiagnosticErrorTrap Trap(Diags); - m_CI->createPCHExternalASTSource(PCHFileName, - DisableValidationForModuleKind::All, - true /*AllowPCHWithCompilerErrors*/, - nullptr /*DeserializationListener*/, - true /*OwnsDeserializationListener*/); - result.push_back(endTransaction(PchT)); - if (Trap.hasErrorOccurred()) { - result.push_back(endTransaction(CurT)); - return false; - } - } + // const std::string& PCHFileName + // = m_CI->getInvocation().getPreprocessorOpts().ImplicitPCHInclude; + // if (!PCHFileName.empty()) { + // Transaction* PchT = beginTransaction(CO); + // DiagnosticErrorTrap Trap(Diags); + // m_CI->createPCHExternalASTSource(PCHFileName, + // DisableValidationForModuleKind::All, + // true /*AllowPCHWithCompilerErrors*/, + // nullptr /*DeserializationListener*/, + // true /*OwnsDeserializationListener*/); + // result.push_back(endTransaction(PchT)); + // if (Trap.hasErrorOccurred()) { + // result.push_back(endTransaction(CurT)); + // return false; + // } + // } addClingPragmas(*m_Interpreter); @@ -856,8 +963,8 @@ namespace cling { // Add the input to the memory buffer, parse it, and add it to the AST. IncrementalParser::EParseResult IncrementalParser::ParseInternal(llvm::StringRef input) { - if (input.empty()) return IncrementalParser::kSuccess; - + if (input.empty()) + return IncrementalParser::kSuccess; Sema& S = getCI()->getSema(); // Recover resources if we crash before exiting this method. diff --git a/interpreter/cling/lib/Interpreter/IncrementalParser.h b/interpreter/cling/lib/Interpreter/IncrementalParser.h index ddea651239262..c13e48c3ada76 100644 --- a/interpreter/cling/lib/Interpreter/IncrementalParser.h +++ b/interpreter/cling/lib/Interpreter/IncrementalParser.h @@ -46,6 +46,7 @@ namespace cling { class Transaction; class TransactionPool; class ASTTransformer; + class IncrementalAction; ///\brief Responsible for the incremental parsing and compilation of input. /// @@ -60,11 +61,14 @@ namespace cling { Interpreter* m_Interpreter; // compiler instance. - std::unique_ptr m_CI; + clang::CompilerInstance *m_CI; // parser (incremental) std::unique_ptr m_Parser; + /// The FrontendAction + IncrementalAction *m_Act; + /// Counts the number of direct user input lines that have been parsed. unsigned InputCount = 0; @@ -112,8 +116,8 @@ namespace cling { }; typedef llvm::PointerIntPair ParseResultTransaction; - IncrementalParser(Interpreter* interp, const char* llvmdir, - const ModuleFileExtensions& moduleExtensions); + IncrementalParser(Interpreter* interp, clang::CompilerInstance* CI, + IncrementalAction* Act); ~IncrementalParser(); ///\brief Whether the IncrementalParser is valid. @@ -124,7 +128,8 @@ namespace cling { bool Initialize(llvm::SmallVectorImpl& result, bool isChildInterpreter); - clang::CompilerInstance* getCI() const { return m_CI.get(); } + // clang::CompilerInstance* getCI() const { return m_CI.get(); } + clang::CompilerInstance* getCI() const { return m_CI; } clang::Parser* getParser() const { return m_Parser.get(); } clang::CodeGenerator* getCodeGenerator() const { return m_CodeGen; } bool hasCodeGenerator() const { return m_CodeGen; } diff --git a/interpreter/cling/lib/Interpreter/Interpreter.cpp b/interpreter/cling/lib/Interpreter/Interpreter.cpp index f9f3767e4d88a..c6863c9754463 100644 --- a/interpreter/cling/lib/Interpreter/Interpreter.cpp +++ b/interpreter/cling/lib/Interpreter/Interpreter.cpp @@ -19,6 +19,7 @@ #include "EnterUserCodeRAII.h" #include "ExternalInterpreterSource.h" #include "ForwardDeclPrinter.h" +#include "IncrementalAction.h" #include "IncrementalExecutor.h" #include "IncrementalParser.h" #include "MultiplexInterpreterCallbacks.h" @@ -219,7 +220,30 @@ namespace cling { auto LLVMCtx = std::make_unique(); TSCtx = std::make_unique(std::move(LLVMCtx)); - m_IncrParser.reset(new IncrementalParser(this, llvmdir, moduleExtensions)); + + m_CI.reset(CIFactory::createCI("\n", getOptions(), llvmdir, std::nullopt, + moduleExtensions)); + + if (!m_CI) { + cling::errs() << "Compiler instance could not be created.\n"; + return; + } + + llvm::Error ErrOut = llvm::Error::success(); + m_Act = + std::make_unique(*m_CI, *getLLVMContext(), + getOptions().CompilerOpts, ErrOut); + + if (ErrOut) { + llvm::logAllUnhandledErrors(std::move(ErrOut), llvm::errs(), + "Action creation failed: "); + return; + } + + m_CI->ExecuteAction(*m_Act); + + m_IncrParser.reset(new IncrementalParser(this, m_CI.get(), m_Act.get())); + if (!m_IncrParser->isValid(false)) return; @@ -281,15 +305,15 @@ namespace cling { bool usingCxxModules = getSema().getLangOpts().Modules; if (usingCxxModules) { - // Explicitly create the modulemanager now. If we would create it later - // implicitly then it would just overwrite our callbacks we set below. - m_IncrParser->getCI()->createASTReader(); - - // When using C++ modules, we setup the callbacks now that we have them - // ready before we parse code for the first time. Without C++ modules - // we can't setup the calls now because the clang PCH currently just - // overwrites it in the Initialize method and we have no simple way to - // initialize them earlier. We handle the non-modules case below. + // // Explicitly create the modulemanager now. If we would create it later + // // implicitly then it would just overwrite our callbacks we set below. + // // m_IncrParser->getCI()->createASTReader(); + + // // When using C++ modules, we setup the callbacks now that we have them + // // ready before we parse code for the first time. Without C++ modules + // // we can't setup the calls now because the clang PCH currently just + // // overwrites it in the Initialize method and we have no simple way to + // // initialize them earlier. We handle the non-modules case below. setupCallbacks(*this, parentInterp); } @@ -302,12 +326,12 @@ namespace cling { } llvm::SmallVector - IncrParserTransactions; + IncrParserTransactions; if (!m_IncrParser->Initialize(IncrParserTransactions, parentInterp)) { // Initialization is not going well, but we still have to commit what // we've been given. Don't clear the DiagnosticsConsumer so the caller // can inspect any errors that have been generated. - for (auto&& I: IncrParserTransactions) + for (auto&& I : IncrParserTransactions) m_IncrParser->commitTransaction(I, false); return; } diff --git a/interpreter/cling/lib/Interpreter/InterpreterCallbacks.cpp b/interpreter/cling/lib/Interpreter/InterpreterCallbacks.cpp index 0281592abdab3..e4c35613d1768 100644 --- a/interpreter/cling/lib/Interpreter/InterpreterCallbacks.cpp +++ b/interpreter/cling/lib/Interpreter/InterpreterCallbacks.cpp @@ -79,6 +79,12 @@ namespace cling { assert(m_Source && "Can't wrap nullptr ExternalASTSource"); } + void InitializeSema(Sema& S) override { + if (ExternalSemaSource* ExternalSema = + dyn_cast_or_null(m_Source)) + ExternalSema->InitializeSema(S); + } + Decl* GetExternalDecl(GlobalDeclID ID) override { return m_Source->GetExternalDecl(ID); } @@ -270,8 +276,11 @@ namespace cling { Sema& SemaRef = interp->getSema(); ASTReader* Reader = m_Interpreter->getCI()->getASTReader().get(); ExternalSemaSource* externalSemaSrc = SemaRef.getExternalSource(); + ExternalASTSource* externalAstSrc = + SemaRef.getASTContext().getExternalSource(); if (enableExternalSemaSourceCallbacks) - if (!externalSemaSrc || externalSemaSrc == Reader) { + if (!externalSemaSrc || externalSemaSrc == Reader || + externalAstSrc == Reader) { // If the ExternalSemaSource is the PCH reader we still need to insert // our listener. m_ExternalSemaSource = new InterpreterExternalSemaSource(this); @@ -304,7 +313,7 @@ namespace cling { // We don't have an existing source, so just set our own source. Ctx.setExternalSource(m_ExternalSemaSource); } - } + } if (enablePPCallbacks) { Preprocessor& PP = m_Interpreter->getCI()->getPreprocessor(); diff --git a/interpreter/llvm-project/clang/include/clang/Frontend/CompilerInstance.h b/interpreter/llvm-project/clang/include/clang/Frontend/CompilerInstance.h index 8b539dfc92960..bfbbb5bac97ce 100644 --- a/interpreter/llvm-project/clang/include/clang/Frontend/CompilerInstance.h +++ b/interpreter/llvm-project/clang/include/clang/Frontend/CompilerInstance.h @@ -203,6 +203,15 @@ class CompilerInstance : public ModuleLoader { /// Force an output buffer. std::unique_ptr OutputStream; +public: + using IncrementalInitFn = + std::function; + +private: + /// A Custom user provided SourceManager initializer for incremental + /// compilation mode. + IncrementalInitFn IncInitFn; + CompilerInstance(const CompilerInstance &) = delete; void operator=(const CompilerInstance &) = delete; public: @@ -486,6 +495,23 @@ class CompilerInstance : public ModuleLoader { /// setSourceManager - Replace the current source manager. void setSourceManager(SourceManager *Value); + /// @} + /// @name IncrementalInitializer + /// @{ + + bool hasIncrementalSourceMgrInitializer() const { + return static_cast(IncInitFn); + } + + void setIncrementalSourceMgrInitializer(IncrementalInitFn Fn) { + IncInitFn = Fn; + } + + bool runIncrementalSourceMgrInitializer(const FrontendInputFile &Input) { + assert(hasIncrementalSourceMgrInitializer()); + return IncInitFn(*this, Input); + } + /// @} /// @name Preprocessor /// @{ diff --git a/interpreter/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp b/interpreter/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp index 7aa3639cabf39..23766873e8092 100644 --- a/interpreter/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp +++ b/interpreter/llvm-project/clang/lib/CodeGen/CodeGenAction.cpp @@ -158,9 +158,9 @@ void BackendConsumer::Initialize(ASTContext &Ctx) { } bool BackendConsumer::HandleTopLevelDecl(DeclGroupRef D) { - PrettyStackTraceDecl CrashInfo(*D.begin(), SourceLocation(), - Context->getSourceManager(), - "LLVM IR generation of declaration"); + // PrettyStackTraceDecl CrashInfo(*D.begin(), SourceLocation(), + // Context->getSourceManager(), + // "LLVM IR generation of declaration"); // Recurse. if (TimerIsEnabled && !LLVMIRGenerationRefCount++) @@ -323,9 +323,9 @@ void BackendConsumer::HandleTranslationUnit(ASTContext &C) { } void BackendConsumer::HandleTagDeclDefinition(TagDecl *D) { - PrettyStackTraceDecl CrashInfo(D, SourceLocation(), - Context->getSourceManager(), - "LLVM IR generation of declaration"); + // PrettyStackTraceDecl CrashInfo(D, SourceLocation(), + // Context->getSourceManager(), + // "LLVM IR generation of declaration"); Gen->HandleTagDeclDefinition(D); } diff --git a/interpreter/llvm-project/clang/lib/Frontend/CompilerInstance.cpp b/interpreter/llvm-project/clang/lib/Frontend/CompilerInstance.cpp index c11c857ea0606..cb12f00c2242b 100644 --- a/interpreter/llvm-project/clang/lib/Frontend/CompilerInstance.cpp +++ b/interpreter/llvm-project/clang/lib/Frontend/CompilerInstance.cpp @@ -1049,7 +1049,8 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) { for (const FrontendInputFile &FIF : getFrontendOpts().Inputs) { // Reset the ID tables if we are reusing the SourceManager and parsing // regular files. - if (hasSourceManager() && !Act.isModelParsingAction()) + if (hasSourceManager() && !Act.isModelParsingAction() && + Act.getTranslationUnitKind() != TU_Incremental) getSourceManager().clearIDTables(); if (Act.BeginSourceFile(*this, FIF)) { diff --git a/interpreter/llvm-project/clang/lib/Frontend/FrontendAction.cpp b/interpreter/llvm-project/clang/lib/Frontend/FrontendAction.cpp index 9f789f093f55d..cd122c174ca55 100644 --- a/interpreter/llvm-project/clang/lib/Frontend/FrontendAction.cpp +++ b/interpreter/llvm-project/clang/lib/Frontend/FrontendAction.cpp @@ -164,6 +164,8 @@ std::unique_ptr FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI, StringRef InFile) { std::unique_ptr Consumer = CreateASTConsumer(CI, InFile); + if (Consumer) + return Consumer; if (!Consumer) return nullptr; @@ -863,7 +865,15 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, CI.getLangOpts().CurrentModule = CI.getLangOpts().ModuleName; } - if (!CI.InitializeSourceManager(Input)) + if (getTranslationUnitKind() == TU_Incremental) { + bool Res = false; + if (CI.hasIncrementalSourceMgrInitializer()) + Res = CI.runIncrementalSourceMgrInitializer(Input); + // else + // Res = CI.initializeDefaultIncrementalSourceMgr(Input); + if (!Res) + return false; + } else if (!CI.InitializeSourceManager(Input)) return false; if (CI.getLangOpts().CPlusPlusModules && Input.getKind().isHeaderUnit() &&