From 817711e0ee46e132d63514a73e8162f1db2afb89 Mon Sep 17 00:00:00 2001 From: PaperChalice Date: Sat, 20 Apr 2024 11:57:31 +0800 Subject: [PATCH] [NewPM][CodeGen] Refactoring `CodeGenPassBuilder` - Remove redundant `std::move`. - Use a unified function to add passes. - Support pass substitution. --- llvm/include/llvm/Passes/CodeGenPassBuilder.h | 817 ++++++++++-------- llvm/include/llvm/Target/TargetMachine.h | 2 +- llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp | 26 +- llvm/lib/Target/X86/X86TargetMachine.h | 2 +- llvm/tools/llc/NewPMDriver.cpp | 2 +- llvm/unittests/CodeGen/CMakeLists.txt | 1 + .../CodeGen/CodeGenPassBuilderTest.cpp | 128 +++ 7 files changed, 594 insertions(+), 384 deletions(-) create mode 100644 llvm/unittests/CodeGen/CodeGenPassBuilderTest.cpp diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index eb15beb835b53..39c5af359db76 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -64,6 +64,7 @@ #include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCTargetOptions.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" @@ -81,6 +82,7 @@ #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include +#include #include #include @@ -121,12 +123,8 @@ namespace llvm { template class CodeGenPassBuilder { public: explicit CodeGenPassBuilder(TargetMachineT &TM, - const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : TM(TM), Opt(Opts), PIC(PIC) { - // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve - // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID) - + const CGPassBuilderOption &Opts, PassBuilder &PB) + : TM(TM), Opt(Opts), PB(PB), PIC(PB.getPassInstrumentationCallbacks()) { // Target should override TM.Options.EnableIPRA in their target-specific // LLVMTM ctor. See TargetMachine::setGlobalISel for example. if (Opt.EnableIPRA) @@ -140,152 +138,106 @@ template class CodeGenPassBuilder { } Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, - raw_pwrite_stream *DwoOut, - CodeGenFileType FileType) const; + raw_pwrite_stream *DwoOut, CodeGenFileType FileType); - PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { - return PIC; + PassInstrumentationCallbacks *getPassInstrumentationCallbacks() { + return PB.getPassInstrumentationCallbacks(); } + /// @brief Substitute PassT with given pass, target can specialize this + /// function template or override it in subclass, if it is overriden by + /// subclass, it must return CodeGenPassBuilder::substitutePass() to get + /// default return value. See also + /// unittests/CodeGen/CodeGenPassBuilderTest.cpp. + /// @tparam PassT The pass type that needs to be replaced. + /// @return The replaced pass if substitution occurs, otherwise return void. + template auto substitutePass() {} + protected: template using has_required_t = decltype(std::declval().isRequired()); - template - using is_module_pass_t = decltype(std::declval().run( - std::declval(), std::declval())); + /// Helper functions with static pass type checking. + /// @{ - template - using is_function_pass_t = decltype(std::declval().run( - std::declval(), std::declval())); - - template - using is_machine_function_pass_t = decltype(std::declval().run( - std::declval(), - std::declval())); + template void addModulePass(PassT &&Pass) { + static_assert(is_module_pass_v, "Must be a module pass!"); + addPass(std::forward(Pass)); + } - // Function object to maintain state while adding codegen IR passes. - // TODO: add a Function -> MachineFunction adaptor and merge - // AddIRPass/AddMachinePass so we can have a function pipeline that runs both - // function passes and machine function passes. - class AddIRPass { - public: - AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {} - ~AddIRPass() { - if (!FPM.isEmpty()) - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); - } + template + void addModulePass(PassT &&Pass, PassTs &&...Passes) { + addModulePass(std::forward(Pass)); + addModulePass(std::forward(Passes)...); + } - template - void operator()(PassT &&Pass, StringRef Name = PassT::name()) { - static_assert((is_detected::value || - is_detected::value) && - "Only module pass and function pass are supported."); - bool Required = false; - if constexpr (is_detected::value) - Required = PassT::isRequired(); - if (!PB.runBeforeAdding(Name) && !Required) - return; - - // Add Function Pass - if constexpr (is_detected::value) { - FPM.addPass(std::forward(Pass)); - } else { - // Add Module Pass - if (!FPM.isEmpty()) { - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); - FPM = FunctionPassManager(); - } + template void addModulePass() { + addModulePass(PassT(), PassTs()...); + } - MPM.addPass(std::forward(Pass)); - } - } + template void addFunctionPass(PassT &&Pass) { + static_assert(is_function_pass_v, "Must be a function pass!"); + addPass(std::forward(Pass)); + } - private: - ModulePassManager &MPM; - FunctionPassManager FPM; - const DerivedT &PB; - }; + template + void addFunctionPass(PassT &&Pass, PassTs &&...Passes) { + addFunctionPass(std::forward(Pass)); + addFunctionPass(std::forward(Passes)...); + } - // Function object to maintain state while adding codegen machine passes. - class AddMachinePass { - public: - AddMachinePass(ModulePassManager &MPM, const DerivedT &PB) - : MPM(MPM), PB(PB) {} - ~AddMachinePass() { - if (!MFPM.isEmpty()) { - FunctionPassManager FPM; - FPM.addPass( - createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); - FPM.addPass(InvalidateAnalysisPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); - } - } + template void addFunctionPass() { + addFunctionPass(PassT(), PassTs()...); + } - template - void operator()(PassT &&Pass, bool Force = false, - StringRef Name = PassT::name()) { - static_assert((is_detected::value || - is_detected::value) && - "Only module pass and function pass are supported."); - - if (!Force && !PB.runBeforeAdding(Name)) - return; - - // Add Function Pass - if constexpr (is_detected::value) { - MFPM.addPass(std::forward(Pass)); - } else { - // Add Module Pass - if (!MFPM.isEmpty()) { - MPM.addPass(createModuleToFunctionPassAdaptor( - createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)))); - MFPM = MachineFunctionPassManager(); - } + template void addMachineFunctionPass(PassT &&Pass) { + static_assert(is_machine_function_pass_v, + "Must be a machine function pass!"); + addPass(std::forward(Pass)); + } - MPM.addPass(std::forward(Pass)); - } + template + void addMachineFunctionPass(PassT &&Pass, PassTs &&...Passes) { + addMachineFunctionPass(std::forward(Pass)); + addMachineFunctionPass(std::forward(Passes)...); + } - for (auto &C : PB.AfterCallbacks) - C(Name, MFPM); - } + template void addMachineFunctionPass() { + addMachineFunctionPass(PassT(), PassTs()...); + } - private: - ModulePassManager &MPM; - MachineFunctionPassManager MFPM; - const DerivedT &PB; - }; + /// @} TargetMachineT &TM; CGPassBuilderOption Opt; + PassBuilder &PB; PassInstrumentationCallbacks *PIC; - template TMC &getTM() const { return static_cast(TM); } - CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } + CodeGenOptLevel getOptLevel() { return TM.getOptLevel(); } /// Check whether or not GlobalISel should abort on error. /// When this is disabled, GlobalISel will fall back on SDISel instead of /// erroring out. - bool isGlobalISelAbortEnabled() const { + bool isGlobalISelAbortEnabled() { return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable; } /// Check whether or not a diagnostic should be emitted when GlobalISel /// uses the fallback path. In other words, it will emit a diagnostic /// when GlobalISel failed and isGlobalISelAbortEnabled is false. - bool reportDiagnosticWhenGlobalISelFallback() const { + bool reportDiagnosticWhenGlobalISelFallback() { return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; } /// addInstSelector - This method should install an instruction selector pass, /// which converts from LLVM code to machine instructions. - Error addInstSelector(AddMachinePass &) const { + Error addInstSelector() { return make_error("addInstSelector is not overridden", inconvertibleErrorCode()); } /// Target can override this to add GlobalMergePass before all IR passes. - void addGlobalMergePass(AddIRPass &) const {} + void addGlobalMergePass() {} /// Add passes that optimize instruction level parallelism for out-of-order /// targets. These passes are run while the machine code is still in SSA @@ -293,11 +245,11 @@ template class CodeGenPassBuilder { /// /// All passes added here should preserve the MachineDominatorTree, /// MachineLoopInfo, and MachineTraceMetrics analyses. - void addILPOpts(AddMachinePass &) const {} + void addILPOpts() {} /// This method may be implemented by targets that want to run passes /// immediately before register allocation. - void addPreRegAlloc(AddMachinePass &) const {} + void addPreRegAlloc() {} /// addPreRewrite - Add passes to the optimized register allocation pipeline /// after register allocation is complete, but before virtual registers are @@ -311,79 +263,77 @@ template class CodeGenPassBuilder { /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not /// be honored. This is also not generally used for the fast variant, /// where the allocation and rewriting are done in one pass. - void addPreRewrite(AddMachinePass &) const {} + void addPreRewrite() {} /// Add passes to be run immediately after virtual registers are rewritten /// to physical registers. - void addPostRewrite(AddMachinePass &) const {} + void addPostRewrite() {} /// This method may be implemented by targets that want to run passes after /// register allocation pass pipeline but before prolog-epilog insertion. - void addPostRegAlloc(AddMachinePass &) const {} + void addPostRegAlloc() {} /// This method may be implemented by targets that want to run passes after /// prolog-epilog insertion and before the second instruction scheduling pass. - void addPreSched2(AddMachinePass &) const {} + void addPreSched2() {} /// This pass may be implemented by targets that want to run passes /// immediately before machine code is emitted. - void addPreEmitPass(AddMachinePass &) const {} + void addPreEmitPass() {} /// Targets may add passes immediately before machine code is emitted in this /// callback. This is called even later than `addPreEmitPass`. // FIXME: Rename `addPreEmitPass` to something more sensible given its actual // position and remove the `2` suffix here as this callback is what // `addPreEmitPass` *should* be but in reality isn't. - void addPreEmitPass2(AddMachinePass &) const {} + void addPreEmitPass2() {} /// {{@ For GlobalISel /// /// addPreISel - This method should add any "last minute" LLVM->LLVM /// passes (which are run just before instruction selector). - void addPreISel(AddIRPass &) const { - llvm_unreachable("addPreISel is not overridden"); - } + void addPreISel() { llvm_unreachable("addPreISel is not overridden"); } /// This method should install an IR translator pass, which converts from /// LLVM code to machine instructions with possibly generic opcodes. - Error addIRTranslator(AddMachinePass &) const { + Error addIRTranslator() { return make_error("addIRTranslator is not overridden", inconvertibleErrorCode()); } /// This method may be implemented by targets that want to run passes /// immediately before legalization. - void addPreLegalizeMachineIR(AddMachinePass &) const {} + void addPreLegalizeMachineIR() {} /// This method should install a legalize pass, which converts the instruction /// sequence into one that can be selected by the target. - Error addLegalizeMachineIR(AddMachinePass &) const { + Error addLegalizeMachineIR() { return make_error("addLegalizeMachineIR is not overridden", inconvertibleErrorCode()); } /// This method may be implemented by targets that want to run passes /// immediately before the register bank selection. - void addPreRegBankSelect(AddMachinePass &) const {} + void addPreRegBankSelect() {} /// This method should install a register bank selector pass, which /// assigns register banks to virtual registers without a register /// class or register banks. - Error addRegBankSelect(AddMachinePass &) const { + Error addRegBankSelect() { return make_error("addRegBankSelect is not overridden", inconvertibleErrorCode()); } /// This method may be implemented by targets that want to run passes /// immediately before the (global) instruction selection. - void addPreGlobalInstructionSelect(AddMachinePass &) const {} + void addPreGlobalInstructionSelect() {} /// This method should install a (global) instruction selector pass, which /// converts possibly generic instructions to fully target-specific /// instructions, thereby constraining all generic virtual registers to /// register classes. - Error addGlobalInstructionSelect(AddMachinePass &) const { + Error addGlobalInstructionSelect() { return make_error( "addGlobalInstructionSelect is not overridden", inconvertibleErrorCode()); @@ -394,30 +344,33 @@ template class CodeGenPassBuilder { /// representation to the MI representation. /// Adds IR based lowering and target specific optimization passes and finally /// the core instruction selection passes. - void addISelPasses(AddIRPass &) const; + Error addISelPasses(bool PrintMIR, raw_pwrite_stream &Out); /// Add the actual instruction selection passes. This does not include /// preparation passes on IR. - Error addCoreISelPasses(AddMachinePass &) const; + Error addCoreISelPasses(); /// Add the complete, standard set of LLVM CodeGen passes. /// Fully developed targets will not generally override this. - Error addMachinePasses(AddMachinePass &) const; + Error addMachinePasses(); /// Add passes to lower exception handling for the code generator. - void addPassesToHandleExceptions(AddIRPass &) const; + void addPassesToHandleExceptions(); /// Add common target configurable passes that perform LLVM IR to IR /// transforms following machine independent optimization. - void addIRPasses(AddIRPass &) const; + void addIRPasses(); + + /// Insertion point in addIRPasses, before adding function passes. + void addTargetIRPasses() {} /// Add pass to prepare the LLVM IR for code generation. This should be done /// before exception handling preparation passes. - void addCodeGenPrepare(AddIRPass &) const; + void addCodeGenPrepare(); /// Add common passes that perform LLVM IR to IR transforms in preparation for /// instruction selection. - void addISelPrepare(AddIRPass &) const; + void addISelPrepare(); /// Methods with trivial inline returns are convenient points in the common /// codegen pass pipeline where targets may insert passes. Methods with @@ -428,30 +381,30 @@ template class CodeGenPassBuilder { /// addMachineSSAOptimization - Add standard passes that optimize machine /// instructions in SSA form. - void addMachineSSAOptimization(AddMachinePass &) const; + void addMachineSSAOptimization(); /// addFastRegAlloc - Add the minimum set of target-independent passes that /// are required for fast register allocation. - Error addFastRegAlloc(AddMachinePass &) const; + Error addFastRegAlloc(); /// addOptimizedRegAlloc - Add passes related to register allocation. /// LLVMTargetMachine provides standard regalloc passes for most targets. - void addOptimizedRegAlloc(AddMachinePass &) const; + void addOptimizedRegAlloc(); /// Add passes that optimize machine instructions after register allocation. - void addMachineLateOptimization(AddMachinePass &) const; + void addMachineLateOptimization(); /// addGCPasses - Add late codegen passes that analyze code for garbage /// collection. This should return true if GC info should be printed after /// these passes. - void addGCPasses(AddMachinePass &) const {} + void addGCPasses() {} /// Add standard basic block placement passes. - void addBlockPlacement(AddMachinePass &) const; + void addBlockPlacement(); using CreateMCStreamer = std::function>(MCContext &)>; - void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const { + void addAsmPrinter(CreateMCStreamer) { llvm_unreachable("addAsmPrinter is not overridden"); } @@ -460,67 +413,235 @@ template class CodeGenPassBuilder { /// createTargetRegisterAllocator - Create the register allocator pass for /// this target at the current optimization level. - void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const; + void addTargetRegisterAllocator(bool Optimized); /// addMachinePasses helper to create the target-selected or overriden /// regalloc pass. - void addRegAllocPass(AddMachinePass &, bool Optimized) const; + void addRegAllocPass(bool Optimized); /// Add core register alloator passes which do the actual register assignment /// and rewriting. \returns true if any passes were added. - Error addRegAssignmentFast(AddMachinePass &) const; - Error addRegAssignmentOptimized(AddMachinePass &) const; + Error addRegAssignmentFast(); + Error addRegAssignmentOptimized(); + + /// Merge all pass manager into one ModulePassManager + void mergePassManager(); /// Allow the target to disable a specific pass by default. /// Backend can declare unwanted passes in constructor. - template void disablePass() { + /// \param InstanceNum - If 0, disable all PassT, else disable only + /// the InstanceNum'th PassT. + template void disablePass(unsigned InstanceNum = 0) { BeforeCallbacks.emplace_back( - [](StringRef Name) { return ((Name != PassTs::name()) && ...); }); + [Cnt = 0u, InstanceNum](StringRef Name) mutable { + if (!InstanceNum) + return PassT::name() != Name; + if (PassT::name() == Name) + return ++Cnt != InstanceNum; + return true; + }); + } + + /// Disable all PassT1, PassT2, PassTs... + template + void disablePass() { + BeforeCallbacks.emplace_back([](StringRef Name) { + return Name != PassT1::name() && Name != PassT2::name() && + ((Name != PassTs::name()) && ...); + }); } /// Insert InsertedPass pass after TargetPass pass. - /// Only machine function passes are supported. + /// If \param InstanceNum - If not 0, insert after each TargetPassT, + /// else only insert after the InstanceNum'th TargetPassT. template - void insertPass(InsertedPassT &&Pass) { + void insertPass(InsertedPassT &&Pass, unsigned InstanceNum = 0) { AfterCallbacks.emplace_back( - [&](StringRef Name, MachineFunctionPassManager &MFPM) mutable { - if (Name == TargetPassT::name()) - MFPM.addPass(std::forward(Pass)); + [Cnt = 0u, InstanceNum, &Pass, this](StringRef Name) mutable { + if (Name == TargetPassT::name()) { + if (!InstanceNum) { + addPass(std::forward(Pass)); + return; + } + if (++Cnt == InstanceNum) + addPass(std::forward(Pass)); + } }); } + ModulePassManager &getMPM() { return MPM; } // Test only! + private: + template + using is_module_pass_t = decltype(std::declval().run( + std::declval(), std::declval())); + + template + static constexpr bool is_module_pass_v = + is_detected::value; + + template + using is_function_pass_t = decltype(std::declval().run( + std::declval(), std::declval())); + + template + static constexpr bool is_function_pass_v = + is_detected::value; + + template + using is_machine_function_pass_t = decltype(std::declval().run( + std::declval(), + std::declval())); + + template + static constexpr bool is_machine_function_pass_v = + is_detected::value; + DerivedT &derived() { return static_cast(*this); } - const DerivedT &derived() const { - return static_cast(*this); + + /// Add one pass to pass manager, it can handle pass nesting automatically. + template void addPass(PassT &&Pass) { + using ResultT = decltype(derived().template substitutePass()); + constexpr bool IsVoid = std::is_void_v; + StringRef PassName = std::conditional_t::name(); + bool Required = false; + if constexpr (is_detected::value) + Required = PassT::isRequired(); + + if (!runBeforeAdding(PassName) || Required) + return; + + if constexpr (IsVoid) + addPassImpl(std::forward(Pass)); + else + addPassImpl(derived().template substitutePass()); + + runAfterAdding(PassName); } - bool runBeforeAdding(StringRef Name) const { + /// A monotonic stack based method to add pass. + template void addPassImpl(PassT &&Pass); + + bool runBeforeAdding(StringRef Name) { bool ShouldAdd = true; for (auto &C : BeforeCallbacks) ShouldAdd &= C(Name); return ShouldAdd; } - void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info) const; + void runAfterAdding(StringRef Name) { + for (auto &C : AfterCallbacks) + C(Name); + } + + void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info); Error verifyStartStop(const TargetPassConfig::StartStopInfo &Info) const; - mutable SmallVector, 4> - BeforeCallbacks; - mutable SmallVector< - llvm::unique_function, 4> - AfterCallbacks; + SmallVector, 4> BeforeCallbacks; + SmallVector, 4> AfterCallbacks; /// Helper variable for `-start-before/-start-after/-stop-before/-stop-after` - mutable bool Started = true; - mutable bool Stopped = true; + bool Started = true; + bool Stopped = true; + + enum class PassType { + ModulePass, + FunctionPass, + MachineFunctionPass, + }; + + template static constexpr bool checkPassType() { + if constexpr (is_module_pass_v) + return T == PassType::ModulePass; + if constexpr (is_function_pass_v) + return T == PassType::FunctionPass; + if constexpr (is_machine_function_pass_v) + return T == PassType::MachineFunctionPass; + } + + std::stack MonoStack; + ModulePassManager MPM; + FunctionPassManager FPM; + MachineFunctionPassManager MFPM; }; +template +template +void CodeGenPassBuilder::addPassImpl(PassT &&Pass) { + static_assert((is_module_pass_v || is_function_pass_v || + is_machine_function_pass_v) && + "Unexpected pass type!"); + + constexpr PassType PT = []() { + if constexpr (std::is_same_v< + InvalidateAnalysisPass, PassT>) + return PassType::FunctionPass; + if constexpr (is_module_pass_v) + return PassType::ModulePass; + if constexpr (is_function_pass_v) + return PassType::FunctionPass; + return PassType::MachineFunctionPass; + }(); + + while (!MonoStack.empty() && MonoStack.top() > PT) { + switch (MonoStack.top()) { + case PassType::MachineFunctionPass: + FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); + MFPM = MachineFunctionPassManager(); + break; + case PassType::FunctionPass: + if (Opt.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + FPM = FunctionPassManager(); + break; + case PassType::ModulePass: + llvm_unreachable("Unexpected pass type!"); + } + MonoStack.pop(); + } + if (MonoStack.empty() || MonoStack.top() < PT) + MonoStack.push(PT); + + if constexpr (PT == PassType::ModulePass) + MPM.addPass(std::forward(Pass)); + else if constexpr (PT == PassType::FunctionPass) + FPM.addPass(std::forward(Pass)); + else + MFPM.addPass(std::forward(Pass)); +} + +template +void CodeGenPassBuilder::mergePassManager() { + if (MonoStack.empty()) + return; + + switch (MonoStack.top()) { + case PassType::MachineFunctionPass: + FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); + [[fallthrough]]; + case PassType::FunctionPass: + if (Opt.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + [[fallthrough]]; + case PassType::ModulePass: + break; + } + MFPM = MachineFunctionPassManager(); + FPM = FunctionPassManager(); + MonoStack = {}; +} + template Error CodeGenPassBuilder::buildPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, - CodeGenFileType FileType) const { + CodeGenFileType FileType) { auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC); if (!StartStopInfo) return StartStopInfo.takeError(); @@ -529,41 +650,38 @@ Error CodeGenPassBuilder::buildPipeline( bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline(); bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null; - { - AddIRPass addIRPass(MPM, derived()); - addIRPass(RequireAnalysisPass()); - addIRPass(RequireAnalysisPass()); - addIRPass(RequireAnalysisPass()); - addISelPasses(addIRPass); - } - - AddMachinePass addPass(MPM, derived()); + addModulePass, + RequireAnalysisPass, + RequireAnalysisPass>(); - if (PrintMIR) - addPass(PrintMIRPreparePass(Out), /*Force=*/true); + derived().addIRPasses(); - if (auto Err = addCoreISelPasses(addPass)) - return std::move(Err); + if (auto Err = addISelPasses(PrintMIR, Out)) + return Err; - if (auto Err = derived().addMachinePasses(addPass)) - return std::move(Err); + if (auto Err = derived().addMachinePasses()) + return Err; if (PrintAsm) { - derived().addAsmPrinter( - addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) { - return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx); - }); + derived().addAsmPrinter([this, &Out, DwoOut, FileType](MCContext &Ctx) { + return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx); + }); } if (PrintMIR) - addPass(PrintMIRPass(Out), /*Force=*/true); + addPassImpl(PrintMIRPass(Out)); - return verifyStartStop(*StartStopInfo); + if (auto Err = verifyStartStop(*StartStopInfo)) + return Err; + addPassImpl(InvalidateAnalysisPass()); + mergePassManager(); + MPM.addPass(std::move(this->MPM)); + return Error::success(); } template void CodeGenPassBuilder::setStartStopPasses( - const TargetPassConfig::StartStopInfo &Info) const { + const TargetPassConfig::StartStopInfo &Info) { if (!Info.StartPass.empty()) { Started = false; BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StartAfter, @@ -622,39 +740,33 @@ Error CodeGenPassBuilder::verifyStartStop( } template -void CodeGenPassBuilder::addISelPasses( - AddIRPass &addPass) const { - derived().addGlobalMergePass(addPass); - if (TM.useEmulatedTLS()) - addPass(LowerEmuTLSPass()); - - addPass(PreISelIntrinsicLoweringPass(&TM)); - addPass(ExpandLargeDivRemPass(&TM)); - addPass(ExpandLargeFpConvertPass(&TM)); - - derived().addIRPasses(addPass); - derived().addCodeGenPrepare(addPass); - addPassesToHandleExceptions(addPass); - derived().addISelPrepare(addPass); +Error CodeGenPassBuilder::addISelPasses( + bool PrintMIR, raw_pwrite_stream &Out) { + derived().addCodeGenPrepare(); + addPassesToHandleExceptions(); + addISelPrepare(); + if (PrintMIR) + addPassImpl(PrintMIRPreparePass(Out)); + return addCoreISelPasses(); } /// Add common target configurable passes that perform LLVM IR to IR transforms /// following machine independent optimization. template -void CodeGenPassBuilder::addIRPasses( - AddIRPass &addPass) const { - // Before running any passes, run the verifier to determine if the input - // coming from the front-end and/or optimizer is valid. - if (!Opt.DisableVerify) - addPass(VerifierPass()); +void CodeGenPassBuilder::addIRPasses() { + derived().addGlobalMergePass(); + if (TM.useEmulatedTLS()) + addModulePass(LowerEmuTLSPass()); + + addModulePass(PreISelIntrinsicLoweringPass(TM)); + + // Add target defined IR passes. + derived().addTargetIRPasses(); // Run loop strength reduction before anything else. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) { - addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), - /*UseMemorySSA=*/true)); - // FIXME: use -stop-after so we could remove PrintLSR - if (Opt.PrintLSR) - addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n")); + addFunctionPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), + /*UseMemorySSA=*/true)); } if (getOptLevel() != CodeGenOptLevel::None) { @@ -663,53 +775,53 @@ void CodeGenPassBuilder::addIRPasses( // into optimally-sized loads and compares. The transforms are enabled by a // target lowering hook. if (!Opt.DisableMergeICmps) - addPass(MergeICmpsPass()); - addPass(ExpandMemCmpPass(&TM)); + addFunctionPass(MergeICmpsPass()); + addFunctionPass(ExpandMemCmpPass(&TM)); } // Run GC lowering passes for builtin collectors - // TODO: add a pass insertion point here - addPass(GCLoweringPass()); - addPass(ShadowStackGCLoweringPass()); - addPass(LowerConstantIntrinsicsPass()); + // TODO: add a pass insertion point and extension point here + addFunctionPass(GCLoweringPass()); + addModulePass(ShadowStackGCLoweringPass()); + addFunctionPass(LowerConstantIntrinsicsPass()); // Make sure that no unreachable blocks are instruction selected. - addPass(UnreachableBlockElimPass()); + addFunctionPass(UnreachableBlockElimPass()); // Prepare expensive constants for SelectionDAG. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting) - addPass(ConstantHoistingPass()); + addFunctionPass(ConstantHoistingPass()); // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector // operands with calls to the corresponding functions in a vector library. if (getOptLevel() != CodeGenOptLevel::None) - addPass(ReplaceWithVeclib()); + addFunctionPass(ReplaceWithVeclib()); if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisablePartialLibcallInlining) - addPass(PartiallyInlineLibCallsPass()); + addFunctionPass(PartiallyInlineLibCallsPass()); // Instrument function entry and exit, e.g. with calls to mcount(). - addPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); + addFunctionPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); // Add scalarization of target's unsupported masked memory intrinsics pass. // the unsupported intrinsic will be replaced with a chain of basic blocks, // that stores/loads element one-by-one if the appropriate mask bit is set. - addPass(ScalarizeMaskedMemIntrinPass()); + addFunctionPass(ScalarizeMaskedMemIntrinPass()); // Expand reduction intrinsics into shuffle sequences if the target wants to. - addPass(ExpandReductionsPass()); + addFunctionPass(ExpandReductionsPass()); // Convert conditional moves to conditional jumps when profitable. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) - addPass(SelectOptimizePass(&TM)); + addFunctionPass(SelectOptimizePass(&TM)); } /// Turn exception handling constructs into something the code generators can /// handle. template -void CodeGenPassBuilder::addPassesToHandleExceptions( - AddIRPass &addPass) const { +void CodeGenPassBuilder::addPassesToHandleExceptions() { const MCAsmInfo *MCAI = TM.getMCAsmInfo(); assert(MCAI && "No MCAsmInfo"); switch (MCAI->getExceptionHandlingType()) { @@ -720,34 +832,33 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( // removed from the parent invoke(s). This could happen when a landing // pad is shared by multiple invokes and is also a target of a normal // edge from elsewhere. - addPass(SjLjEHPreparePass(&TM)); + addFunctionPass(SjLjEHPreparePass(&TM)); [[fallthrough]]; case ExceptionHandling::DwarfCFI: case ExceptionHandling::ARM: case ExceptionHandling::AIX: case ExceptionHandling::ZOS: - addPass(DwarfEHPreparePass(&TM)); + addFunctionPass(DwarfEHPreparePass(&TM)); break; case ExceptionHandling::WinEH: // We support using both GCC-style and MSVC-style exceptions on Windows, so // add both preparation passes. Each pass will only actually run if it // recognizes the personality function. - addPass(WinEHPreparePass()); - addPass(DwarfEHPreparePass(&TM)); + addFunctionPass(WinEHPreparePass(), DwarfEHPreparePass(&TM)); break; case ExceptionHandling::Wasm: // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs // on catchpads and cleanuppads because it does not outline them into // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we // should remove PHIs there. - addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false)); - addPass(WasmEHPreparePass()); + addFunctionPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false), + WasmEHPreparePass()); break; case ExceptionHandling::None: - addPass(LowerInvokePass()); + addFunctionPass(LowerInvokePass()); // The lower invoke pass may create unreachable code. Remove it. - addPass(UnreachableBlockElimPass()); + addFunctionPass(UnreachableBlockElimPass()); break; } } @@ -755,10 +866,9 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( /// Add pass to prepare the LLVM IR for code generation. This should be done /// before exception handling preparation passes. template -void CodeGenPassBuilder::addCodeGenPrepare( - AddIRPass &addPass) const { +void CodeGenPassBuilder::addCodeGenPrepare() { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) - addPass(CodeGenPreparePass(&TM)); + addFunctionPass(CodeGenPreparePass(&TM)); // TODO: Default ctor'd RewriteSymbolPass is no-op. // addPass(RewriteSymbolPass()); } @@ -766,29 +876,21 @@ void CodeGenPassBuilder::addCodeGenPrepare( /// Add common passes that perform LLVM IR to IR transforms in preparation for /// instruction selection. template -void CodeGenPassBuilder::addISelPrepare( - AddIRPass &addPass) const { - derived().addPreISel(addPass); +void CodeGenPassBuilder::addISelPrepare() { + derived().addPreISel(); - addPass(CallBrPreparePass()); + addFunctionPass(CallBrPreparePass()); // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. - addPass(SafeStackPass(&TM)); - addPass(StackProtectorPass(&TM)); + addFunctionPass(SafeStackPass(&TM), StackProtectorPass(&TM)); if (Opt.PrintISelInput) - addPass(PrintFunctionPass(dbgs(), - "\n\n*** Final LLVM Code input to ISel ***\n")); - - // All passes which modify the LLVM IR are now complete; run the verifier - // to ensure that the IR is valid. - if (!Opt.DisableVerify) - addPass(VerifierPass()); + addFunctionPass(PrintFunctionPass( + dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n")); } template -Error CodeGenPassBuilder::addCoreISelPasses( - AddMachinePass &addPass) const { +Error CodeGenPassBuilder::addCoreISelPasses() { // Enable FastISel with -fast-isel, but allow that to be overridden. TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true)); @@ -820,42 +922,42 @@ Error CodeGenPassBuilder::addCoreISelPasses( // Add instruction selector passes. if (Selector == SelectorType::GlobalISel) { - if (auto Err = derived().addIRTranslator(addPass)) - return std::move(Err); + if (auto Err = derived().addIRTranslator()) + return Err; - derived().addPreLegalizeMachineIR(addPass); + derived().addPreLegalizeMachineIR(); - if (auto Err = derived().addLegalizeMachineIR(addPass)) - return std::move(Err); + if (auto Err = derived().addLegalizeMachineIR()) + return Err; // Before running the register bank selector, ask the target if it // wants to run some passes. - derived().addPreRegBankSelect(addPass); + derived().addPreRegBankSelect(); - if (auto Err = derived().addRegBankSelect(addPass)) - return std::move(Err); + if (auto Err = derived().addRegBankSelect()) + return Err; - derived().addPreGlobalInstructionSelect(addPass); + derived().addPreGlobalInstructionSelect(); - if (auto Err = derived().addGlobalInstructionSelect(addPass)) - return std::move(Err); + if (auto Err = derived().addGlobalInstructionSelect()) + return Err; // Pass to reset the MachineFunction if the ISel failed. - addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(), - isGlobalISelAbortEnabled())); + addMachineFunctionPass(ResetMachineFunctionPass( + reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled())); // Provide a fallback path when we do not want to abort on // not-yet-supported input. if (!isGlobalISelAbortEnabled()) - if (auto Err = derived().addInstSelector(addPass)) - return std::move(Err); + if (auto Err = derived().addInstSelector()) + return Err; - } else if (auto Err = derived().addInstSelector(addPass)) - return std::move(Err); + } else if (auto Err = derived().addInstSelector()) + return Err; // Expand pseudo-instructions emitted by ISel. Don't run the verifier before // FinalizeISel. - addPass(FinalizeISelPass()); + addMachineFunctionPass(FinalizeISelPass()); // // Print the instruction selected machine code... // printAndVerify("After Instruction Selection"); @@ -880,94 +982,90 @@ Error CodeGenPassBuilder::addCoreISelPasses( /// dependencies on multiple passes, the target should override the stage /// instead. template -Error CodeGenPassBuilder::addMachinePasses( - AddMachinePass &addPass) const { +Error CodeGenPassBuilder::addMachinePasses() { // Add passes that optimize machine instructions in SSA form. if (getOptLevel() != CodeGenOptLevel::None) { - derived().addMachineSSAOptimization(addPass); + derived().addMachineSSAOptimization(); } else { // If the target requests it, assign local variables to stack slots relative // to one another and simplify frame index references where possible. - addPass(LocalStackSlotAllocationPass()); + addMachineFunctionPass(LocalStackSlotPass()); } if (TM.Options.EnableIPRA) - addPass(RegUsageInfoPropagationPass()); + addMachineFunctionPass(RegUsageInfoPropagationPass()); // Run pre-ra passes. - derived().addPreRegAlloc(addPass); + derived().addPreRegAlloc(); // Run register allocation and passes that are tightly coupled with it, // including phi elimination and scheduling. if (*Opt.OptimizeRegAlloc) { - derived().addOptimizedRegAlloc(addPass); + derived().addOptimizedRegAlloc(); } else { - if (auto Err = derived().addFastRegAlloc(addPass)) + if (auto Err = derived().addFastRegAlloc()) return Err; } // Run post-ra passes. - derived().addPostRegAlloc(addPass); + derived().addPostRegAlloc(); - addPass(RemoveRedundantDebugValuesPass()); + addMachineFunctionPass(RemoveRedundantDebugValuesPass()); // Insert prolog/epilog code. Eliminate abstract frame index references... - if (getOptLevel() != CodeGenOptLevel::None) { - addPass(PostRAMachineSinkingPass()); - addPass(ShrinkWrapPass()); - } + if (getOptLevel() != CodeGenOptLevel::None) + addMachineFunctionPass(PostRAMachineSinkingPass(), ShrinkWrapPass()); + + addMachineFunctionPass(PrologEpilogInserterPass()); - addPass(PrologEpilogInserterPass()); + // TODO: Add extension point here. /// Add passes that optimize machine instructions after register allocation. if (getOptLevel() != CodeGenOptLevel::None) - derived().addMachineLateOptimization(addPass); + derived().addMachineLateOptimization(); // Expand pseudo instructions before second scheduling pass. - addPass(ExpandPostRAPseudosPass()); + addMachineFunctionPass(ExpandPostRAPseudosPass()); // Run pre-sched2 passes. - derived().addPreSched2(addPass); + derived().addPreSched2(); if (Opt.EnableImplicitNullChecks) - addPass(ImplicitNullChecksPass()); + addMachineFunctionPass(ImplicitNullChecksPass()); // Second pass scheduler. // Let Target optionally insert this pass by itself at some other // point. + // TODO: Migrate to PostMachineSchedulerPass. if (getOptLevel() != CodeGenOptLevel::None && !TM.targetSchedulesPostRAScheduling()) { if (Opt.MISchedPostRA) - addPass(PostMachineSchedulerPass()); + addMachineFunctionPass(PostMachineSchedulerPass()); else - addPass(PostRASchedulerPass()); + addMachineFunctionPass(PostRASchedulerPass()); } // GC - derived().addGCPasses(addPass); + derived().addGCPasses(); // Basic block placement. if (getOptLevel() != CodeGenOptLevel::None) - derived().addBlockPlacement(addPass); + derived().addBlockPlacement(); // Insert before XRay Instrumentation. - addPass(FEntryInserterPass()); + addMachineFunctionPass(FEntryInserterPass(), XRayInstrumentationPass(), + PatchableFunctionPass()); - addPass(XRayInstrumentationPass()); - addPass(PatchableFunctionPass()); - - derived().addPreEmitPass(addPass); + derived().addPreEmitPass(); if (TM.Options.EnableIPRA) // Collect register usage information and produce a register mask of // clobbered registers, to be used to optimize call sites. - addPass(RegUsageInfoCollectorPass()); - - addPass(FuncletLayoutPass()); + addMachineFunctionPass(RegUsageInfoCollectorPass()); - addPass(StackMapLivenessPass()); - addPass(LiveDebugValuesPass()); - addPass(MachineSanitizerBinaryMetadata()); + addMachineFunctionPass(FuncletLayoutPass(), StackMapLivenessPass(), + LiveDebugValuesPass(), + MachineSanitizerBinaryMetadata()); if (TM.Options.EnableMachineOutliner && getOptLevel() != CodeGenOptLevel::None && @@ -976,54 +1074,50 @@ Error CodeGenPassBuilder::addMachinePasses( (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline); bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining; if (AddOutliner) - addPass(MachineOutlinerPass(RunOnAllFunctions)); + addModulePass(MachineOutlinerPass(RunOnAllFunctions)); } // Add passes that directly emit MI after all other MI passes. - derived().addPreEmitPass2(addPass); + derived().addPreEmitPass2(); return Error::success(); } /// Add passes that optimize machine instructions in SSA form. template -void CodeGenPassBuilder::addMachineSSAOptimization( - AddMachinePass &addPass) const { +void CodeGenPassBuilder::addMachineSSAOptimization() { // Pre-ra tail duplication. - addPass(EarlyTailDuplicatePass()); + addMachineFunctionPass(EarlyTailDuplicatePass()); // Optimize PHIs before DCE: removing dead PHI cycles may make more // instructions dead. - addPass(OptimizePHIsPass()); + addMachineFunctionPass(OptimizePHIsPass()); // This pass merges large allocas. StackSlotColoring is a different pass // which merges spill slots. - addPass(StackColoringPass()); + addMachineFunctionPass(StackColoringPass()); // If the target requests it, assign local variables to stack slots relative // to one another and simplify frame index references where possible. - addPass(LocalStackSlotAllocationPass()); + addMachineFunctionPass(LocalStackSlotPass()); // With optimization, dead code should already be eliminated. However // there is one known exception: lowered code for arguments that are only // used by tail calls, where the tail calls reuse the incoming stack // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). - addPass(DeadMachineInstructionElimPass()); + addMachineFunctionPass(DeadMachineInstructionElimPass()); // Allow targets to insert passes that improve instruction level parallelism, // like if-conversion. Such passes will typically need dominator trees and // loop info, just like LICM and CSE below. - derived().addILPOpts(addPass); + derived().addILPOpts(); - addPass(EarlyMachineLICMPass()); - addPass(MachineCSEPass()); + addMachineFunctionPass(); - addPass(MachineSinkingPass()); - - addPass(PeepholeOptimizerPass()); // Clean-up the dead code that may have been generated by peephole // rewriting. - addPass(DeadMachineInstructionElimPass()); + addMachineFunctionPass(DeadMachineInstructionElimPass()); } //===---------------------------------------------------------------------===// @@ -1040,11 +1134,11 @@ void CodeGenPassBuilder::addMachineSSAOptimization( /// selection. But -regalloc=... always takes precedence. template void CodeGenPassBuilder::addTargetRegisterAllocator( - AddMachinePass &addPass, bool Optimized) const { + bool Optimized) { if (Optimized) - addPass(RAGreedyPass()); + addMachineFunctionPass(RAGreedyPass()); else - addPass(RegAllocFastPass()); + addMachineFunctionPass(RegAllocFastPass()); } /// Find and instantiate the register allocation pass requested by this target @@ -1052,34 +1146,32 @@ void CodeGenPassBuilder::addTargetRegisterAllocator( /// defined as separate passes because they may require different analysis. template void CodeGenPassBuilder::addRegAllocPass( - AddMachinePass &addPass, bool Optimized) const { + bool Optimized) { // TODO: Parse Opt.RegAlloc to add register allocator. } template -Error CodeGenPassBuilder::addRegAssignmentFast( - AddMachinePass &addPass) const { +Error CodeGenPassBuilder::addRegAssignmentFast() { // TODO: Ensure allocator is default or fast. - addRegAllocPass(addPass, false); + addRegAllocPass(false); return Error::success(); } template -Error CodeGenPassBuilder::addRegAssignmentOptimized( - AddMachinePass &addPass) const { +Error CodeGenPassBuilder::addRegAssignmentOptimized() { // Add the selected register allocation pass. - addRegAllocPass(addPass, true); + addRegAllocPass(true); // Allow targets to change the register assignments before rewriting. - derived().addPreRewrite(addPass); + derived().addPreRewrite(); // Finally rewrite virtual registers. - addPass(VirtRegRewriterPass()); + addMachineFunctionPass(VirtRegRewriterPass()); // Perform stack slot coloring and post-ra machine LICM. // // FIXME: Re-enable coloring with register when it's capable of adding // kill markers. - addPass(StackSlotColoringPass()); + addMachineFunctionPass(StackSlotColoringPass()); return Error::success(); } @@ -1087,56 +1179,49 @@ Error CodeGenPassBuilder::addRegAssignmentOptimized( /// Add the minimum set of target-independent passes that are required for /// register allocation. No coalescing or scheduling. template -Error CodeGenPassBuilder::addFastRegAlloc( - AddMachinePass &addPass) const { - addPass(PHIEliminationPass()); - addPass(TwoAddressInstructionPass()); - return derived().addRegAssignmentFast(addPass); +Error CodeGenPassBuilder::addFastRegAlloc() { + addMachineFunctionPass(PHIEliminationPass(), TwoAddressInstructionPass()); + return derived().addRegAssignmentFast(); } /// Add standard target-independent passes that are tightly coupled with /// optimized register allocation, including coalescing, machine instruction /// scheduling, and register allocation itself. template -void CodeGenPassBuilder::addOptimizedRegAlloc( - AddMachinePass &addPass) const { - addPass(DetectDeadLanesPass()); - - addPass(InitUndefPass()); - - addPass(ProcessImplicitDefsPass()); +void CodeGenPassBuilder::addOptimizedRegAlloc() { + addMachineFunctionPass(); // Edge splitting is smarter with machine loop info. - addPass(PHIEliminationPass()); + addMachineFunctionPass(PHIEliminationPass()); // Eventually, we want to run LiveIntervals before PHI elimination. if (Opt.EarlyLiveIntervals) - addPass(RequireAnalysisPass()); + addMachineFunctionPass(LiveIntervalsPass()); - addPass(TwoAddressInstructionPass()); - addPass(RegisterCoalescerPass()); + addMachineFunctionPass(); // The machine scheduler may accidentally create disconnected components // when moving subregister definitions around, avoid this by splitting them to // separate vregs before. Splitting can also improve reg. allocation quality. - addPass(RenameIndependentSubregsPass()); + addMachineFunctionPass(RenameIndependentSubregsPass()); // PreRA instruction scheduling. - addPass(MachineSchedulerPass()); + addMachineFunctionPass(MachineSchedulerPass()); - if (derived().addRegAssignmentOptimized(addPass)) { + if (derived().addRegAssignmentOptimized()) { // Allow targets to expand pseudo instructions depending on the choice of // registers before MachineCopyPropagation. - derived().addPostRewrite(addPass); + derived().addPostRewrite(); // Copy propagate to forward register uses and try to eliminate COPYs that // were not coalesced. - addPass(MachineCopyPropagationPass()); + addMachineFunctionPass(MachineCopyPropagationPass()); // Run post-ra machine LICM to hoist reloads / remats. // // FIXME: can this move into MachineLateOptimization? - addPass(MachineLICMPass()); + addMachineFunctionPass(MachineLICMPass()); } } @@ -1146,33 +1231,31 @@ void CodeGenPassBuilder::addOptimizedRegAlloc( /// Add passes that optimize machine instructions after register allocation. template -void CodeGenPassBuilder::addMachineLateOptimization( - AddMachinePass &addPass) const { +void CodeGenPassBuilder::addMachineLateOptimization() { // Branch folding must be run after regalloc and prolog/epilog insertion. - addPass(BranchFolderPass()); + addMachineFunctionPass(BranchFolderPass()); // Tail duplication. // Note that duplicating tail just increases code size and degrades // performance for targets that require Structured Control Flow. // In addition it can also make CFG irreducible. Thus we disable it. if (!TM.requiresStructuredCFG()) - addPass(TailDuplicatePass()); + addMachineFunctionPass(TailDuplicatePass()); // Cleanup of redundant (identical) address/immediate loads. - addPass(MachineLateInstrsCleanupPass()); + addMachineFunctionPass(MachineLateInstrsCleanupPass()); // Copy propagation. - addPass(MachineCopyPropagationPass()); + addMachineFunctionPass(MachineCopyPropagationPass()); } /// Add standard basic block placement passes. template -void CodeGenPassBuilder::addBlockPlacement( - AddMachinePass &addPass) const { - addPass(MachineBlockPlacementPass()); +void CodeGenPassBuilder::addBlockPlacement() { + addMachineFunctionPass(MachineBlockPlacementPass()); // Run a separate pass to collect block placement statistics. if (Opt.EnableBlockPlacementStats) - addPass(MachineBlockPlacementStatsPass()); + addMachineFunctionPass(MachineBlockPlacementStatsPass()); } } // namespace llvm diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index c3e9d41315f61..bbf66df5e255c 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -472,7 +472,7 @@ class LLVMTargetMachine : public TargetMachine { virtual Error buildCodeGenPipeline(ModulePassManager &, raw_pwrite_stream &, raw_pwrite_stream *, CodeGenFileType, const CGPassBuilderOption &, - PassInstrumentationCallbacks *) { + PassBuilder &) { return make_error("buildCodeGenPipeline is not overridden", inconvertibleErrorCode()); } diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp index d979517e12af6..987fcea0a39a4 100644 --- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp +++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp @@ -26,25 +26,24 @@ class X86CodeGenPassBuilder public: explicit X86CodeGenPassBuilder(X86TargetMachine &TM, const CGPassBuilderOption &Opts, - PassInstrumentationCallbacks *PIC) - : CodeGenPassBuilder(TM, Opts, PIC) {} - void addPreISel(AddIRPass &addPass) const; - void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; - Error addInstSelector(AddMachinePass &) const; + PassBuilder &PB) + : CodeGenPassBuilder(TM, Opts, PB) {} + void addPreISel(); + void addAsmPrinter(CreateMCStreamer); + Error addInstSelector(); }; -void X86CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const { +void X86CodeGenPassBuilder::addPreISel() { // TODO: Add passes pre instruction selection. } -void X86CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass, - CreateMCStreamer) const { +void X86CodeGenPassBuilder::addAsmPrinter(CreateMCStreamer) { // TODO: Add AsmPrinter. } -Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { - // TODO: Add instruction selector related passes. - addPass(X86ISelDAGToDAGPass(TM)); +Error X86CodeGenPassBuilder::addInstSelector() { + // TODO: Add instruction selector. + addMachineFunctionPass(X86ISelDAGToDAGPass(TM)); return Error::success(); } @@ -57,8 +56,7 @@ void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { Error X86TargetMachine::buildCodeGenPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, - CodeGenFileType FileType, const CGPassBuilderOption &Opt, - PassInstrumentationCallbacks *PIC) { - auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC); + CodeGenFileType FileType, const CGPassBuilderOption &Opt, PassBuilder &PB) { + auto CGPB = X86CodeGenPassBuilder(*this, Opt, PB); return CGPB.buildPipeline(MPM, Out, DwoOut, FileType); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index ec4a93e9c9d4b..0edbad7557918 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -71,7 +71,7 @@ class X86TargetMachine final : public LLVMTargetMachine { Error buildCodeGenPipeline(ModulePassManager &, raw_pwrite_stream &, raw_pwrite_stream *, CodeGenFileType, const CGPassBuilderOption &, - PassInstrumentationCallbacks *) override; + PassBuilder &) override; bool isJIT() const { return IsJIT; } diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp index c8088da49a278..7144df4afd926 100644 --- a/llvm/tools/llc/NewPMDriver.cpp +++ b/llvm/tools/llc/NewPMDriver.cpp @@ -155,7 +155,7 @@ int llvm::compileModuleWithNewPM( return 1; } else { ExitOnErr(LLVMTM.buildCodeGenPipeline( - MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC)); + MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, PB)); } if (PrintPipelinePasses) { diff --git a/llvm/unittests/CodeGen/CMakeLists.txt b/llvm/unittests/CodeGen/CMakeLists.txt index 963cdcc0275e1..568ce5dfda645 100644 --- a/llvm/unittests/CodeGen/CMakeLists.txt +++ b/llvm/unittests/CodeGen/CMakeLists.txt @@ -25,6 +25,7 @@ add_llvm_unittest(CodeGenTests AMDGPUMetadataTest.cpp AsmPrinterDwarfTest.cpp CCStateTest.cpp + CodeGenPassBuilderTest.cpp DIEHashTest.cpp DIETest.cpp DwarfStringPoolEntryRefTest.cpp diff --git a/llvm/unittests/CodeGen/CodeGenPassBuilderTest.cpp b/llvm/unittests/CodeGen/CodeGenPassBuilderTest.cpp new file mode 100644 index 0000000000000..073ac82c0ae1a --- /dev/null +++ b/llvm/unittests/CodeGen/CodeGenPassBuilderTest.cpp @@ -0,0 +1,128 @@ +//===- llvm/unittest/CodeGen/CodeGenPassBuilderTest.cpp -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Passes/CodeGenPassBuilder.h" +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Target/TargetMachine.h" + +#include "gtest/gtest.h" + +using namespace llvm; + +namespace { + +class TestTargetMachine : public LLVMTargetMachine { +public: + TestTargetMachine() + : LLVMTargetMachine(Target(), "", Triple(""), "", "", TargetOptions(), + Reloc::Static, CodeModel::Small, + CodeGenOptLevel::Default) {} +}; + +TestTargetMachine &createTargetMachine() { + static TestTargetMachine TM; + return TM; +} + +struct DisabledMachineFunctionPass + : public PassInfoMixin { + PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { + return PreservedAnalyses::all(); + } +}; + +struct ReplacedMachineFunctionPass + : public PassInfoMixin { + PreservedAnalyses run(MachineFunction &, MachineFunctionAnalysisManager &) { + return PreservedAnalyses::all(); + } +}; + +class TestCodeGenPassBuilder + : public CodeGenPassBuilder { + +public: + explicit TestCodeGenPassBuilder(PassBuilder &PB) + : CodeGenPassBuilder(createTargetMachine(), CGPassBuilderOption(), PB) { + // Declare disabled passes in constructor. + disablePass(3); // Disable the third NoOpModulePass. + disablePass(); + } + + // Override substitutePass is also OK. + // template auto substitutePass() { + // if constexpr (std::is_same_v) + // return NoOpMachineFunctionPass(); + // else + // return; + // } + + void buildTestPipeline(ModulePassManager &MPM) { + addModulePass(); + addFunctionPass(); + addModulePass(); + addMachineFunctionPass(); + addFunctionPass(NoOpFunctionPass()); + addMachineFunctionPass(); + mergePassManager(); + MPM.addPass(std::move(getMPM())); + getMPM() = ModulePassManager(); + } +}; + +class CodeGenPassBuilderTest : public testing::Test { +public: + CodeGenPassBuilderTest() : PB(&createTargetMachine()) { + PIC.addClassToPassName(NoOpModulePass::name(), "no-op-module"); + PIC.addClassToPassName(NoOpFunctionPass::name(), "no-op-function"); + PIC.addClassToPassName(NoOpMachineFunctionPass::name(), + "no-op-machine-function"); + PIC.addClassToPassName(DisabledMachineFunctionPass::name(), "disabled"); + PIC.addClassToPassName(ReplacedMachineFunctionPass::name(), "replaced"); + } + + void buildPipeline(ModulePassManager &MPM) { + TestCodeGenPassBuilder CGPB(PB); + CGPB.buildTestPipeline(MPM); + } + + std::string getPipelineText(ModulePassManager &MPM) { + std::string PipelineText; + raw_string_ostream OS(PipelineText); + MPM.printPipeline( + OS, [&](StringRef S) { return PIC.getPassNameForClassName(S); }); + return PipelineText; + } + PassInstrumentationCallbacks PIC; + PassBuilder PB; +}; + +} // namespace + +using PassBuilderBase = + CodeGenPassBuilder; + +// Add a specialization to substitute a pass. +template <> +template <> +auto PassBuilderBase::substitutePass() { + return NoOpMachineFunctionPass(); +} + +TEST_F(CodeGenPassBuilderTest, Basic) { + ModulePassManager MPM; + buildPipeline(MPM); + EXPECT_EQ(getPipelineText(MPM), + "no-op-module,no-op-module,function(no-op-function,no-op-function," + "machine-function(no-op-machine-function,no-op-machine-function))"); +}