diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h index 66c79c74f2be5..5a5cd24ee577b 100644 --- a/llvm/include/llvm/CodeGen/TargetPassConfig.h +++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h @@ -190,8 +190,7 @@ class TargetPassConfig : public ImmutablePass { /// Returns pass name in `-stop-before` or `-stop-after` /// NOTE: New pass manager migration only - static Expected - getStartStopInfo(PassInstrumentationCallbacks &PIC); + static Expected getStartStopInfo(); void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); } diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index d2e9e8185a2b9..182d70959394a 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -534,7 +534,7 @@ template Error CodeGenPassBuilder::buildPipeline( ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const { - auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC); + auto StartStopInfo = TargetPassConfig::getStartStopInfo(); if (!StartStopInfo) return StartStopInfo.takeError(); setStartStopPasses(*StartStopInfo); diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 851561f6b769b..f730125681108 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -206,6 +206,7 @@ DUMMY_MACHINE_MODULE_PASS("mir-strip-debug", StripDebugMachineModulePass) #ifndef DUMMY_MACHINE_FUNCTION_PASS #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME) #endif +DUMMY_MACHINE_FUNCTION_PASS("bb-path-cloning", BasicBlockPathCloningPass) DUMMY_MACHINE_FUNCTION_PASS("bbsections-prepare", BasicBlockSectionsPass) DUMMY_MACHINE_FUNCTION_PASS("bbsections-profile-reader", BasicBlockSectionsProfileReaderPass) DUMMY_MACHINE_FUNCTION_PASS("block-placement", MachineBlockPlacementPass) @@ -222,6 +223,8 @@ DUMMY_MACHINE_FUNCTION_PASS("fixup-statepoint-caller-saved", FixupStatepointCall DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass) DUMMY_MACHINE_FUNCTION_PASS("funclet-layout", FuncletLayoutPass) DUMMY_MACHINE_FUNCTION_PASS("gc-empty-basic-blocks", GCEmptyBasicBlocksPass) +DUMMY_MACHINE_FUNCTION_PASS("gc-machine-code-insersion", + GCMachineCodeInsertionPass) DUMMY_MACHINE_FUNCTION_PASS("implicit-null-checks", ImplicitNullChecksPass) DUMMY_MACHINE_FUNCTION_PASS("init-undef-pass", InitUndefPass) DUMMY_MACHINE_FUNCTION_PASS("instruction-select", InstructionSelectPass) @@ -266,6 +269,8 @@ DUMMY_MACHINE_FUNCTION_PASS("stack-frame-layout", StackFrameLayoutAnalysisPass) DUMMY_MACHINE_FUNCTION_PASS("stack-slot-coloring", StackSlotColoringPass) DUMMY_MACHINE_FUNCTION_PASS("stackmap-liveness", StackMapLivenessPass) DUMMY_MACHINE_FUNCTION_PASS("unpack-mi-bundles", UnpackMachineBundlesPass) +DUMMY_MACHINE_FUNCTION_PASS("unreachable-mbb-elimination", + UnreachableMachineBlockElimPass) DUMMY_MACHINE_FUNCTION_PASS("virtregrewriter", VirtRegRewriterPass) DUMMY_MACHINE_FUNCTION_PASS("xray-instrumentation", XRayInstrumentationPass) #undef DUMMY_MACHINE_FUNCTION_PASS diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index e7bc3a58f414f..dfb27925db722 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -15,14 +15,17 @@ #ifndef LLVM_PASSES_PASSBUILDER_H #define LLVM_PASSES_PASSBUILDER_H +#include "llvm/ADT/StringSet.h" #include "llvm/Analysis/CGSCCPassManager.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/RegAllocCommon.h" #include "llvm/IR/PassManager.h" #include "llvm/Passes/OptimizationLevel.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/PGOOptions.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Transforms/IPO/Inliner.h" #include "llvm/Transforms/IPO/ModuleInliner.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" @@ -34,6 +37,7 @@ class StringRef; class AAManager; class TargetMachine; class ModuleSummaryIndex; +class MCContext; template class IntrusiveRefCntPtr; namespace vfs { class FileSystem; @@ -106,6 +110,7 @@ class PassBuilder { TargetMachine *TM; PipelineTuningOptions PTO; std::optional PGOOpt; + CGPassBuilderOption CGPBO; PassInstrumentationCallbacks *PIC; public: @@ -308,6 +313,24 @@ class PassBuilder { /// TargetMachine::registerDefaultAliasAnalyses(). AAManager buildDefaultAAPipeline(); + /// Build CodeGen pass pipeline. + /// + /// {{@ + Expected + buildDefaultCodeGenPipeline(raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, + CodeGenFileType FileType, MCContext &Ctx); + Error buildDefaultCodeGenPipeline(ModulePassManager &MPM, + raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, + CodeGenFileType FileType, MCContext &Ctx); + Error addRegAllocPass(MachineFunctionPassManager &MFPM, + StringRef Filter = "all"); + // TODO: Add method to build MC emission pipeline. + template void disablePass() { + (DisabledPasses.insert(PassTs::name()), ...); + } + /// @}} + /// Parse a textual pass pipeline description into a \c /// ModulePassManager. /// @@ -523,6 +546,133 @@ class PassBuilder { FullLinkTimeOptimizationLastEPCallbacks.push_back(C); } + /// Register target specific callbacks to extend codegen pipeline. + /// {{@ + + /// If target want its own pipeline, use this callback. + void setCustomCodeGenPipelineBuilderCallback( + const std::function + C) { + CustomCodeGenPipelineBuilderCallback = C; + } + + void registerCodeGenIREarlyEPCallback( + const std::function C) { + CodeGenIREarlyEPCallbacks.push_back(C); + } + + void registerGCLoweringEPCallback( + const std::function C) { + GCLoweringEPCallbacks.push_back(C); + } + + void registerISelPrepareEPCallback( + const std::function &C) { + ISelPrepareEPCallbacks.push_back(C); + } + + void registerMachineSSAOptimizationEarlyEPCallback( + const std::function &C) { + MachineSSAOptimizationEarlyEPCallbacks.push_back(C); + } + + void registerILPOptsEPCallback( + const std::function &C) { + ILPOptsEPCallbacks.push_back(C); + } + + void registerMachineSSAOptimizationLastEPCallback( + const std::function &C) { + MachineSSAOptimizationLastEPCallbacks.push_back(C); + } + + void registerPreRegAllocEPCallback( + const std::function &C) { + PreRegAllocEPCallbacks.push_back(C); + } + + void registerPostRegAllocEPCallback( + const std::function &C) { + PostRegAllocEPCallbacks.push_back(C); + } + + void registerPreRegBankSelectEPCallback( + const std::function &C) { + PreRegBankSelectEPCallbacks.push_back(C); + } + + void registerPreGlobalInstructionSelectEPCallback( + const std::function &C) { + PreGlobalInstructionSelectEPCallbacks.push_back(C); + } + + void registerPostGlobalInstructionSelectEPCallback( + const std::function &C) { + PostGlobalInstructionSelectEPCallbacks.push_back(C); + } + + void registerMachineLateOptimizationEPCallback( + const std::function &C) { + MachineLateOptimizationEPCallbacks.push_back(C); + } + + void registerPreSched2EPCallback( + const std::function &C) { + PreSched2EPCallbacks.push_back(C); + } + + void registerPostRewriteEPCallback( + const std::function &C) { + PostRewriteEPCallbacks.push_back(C); + } + + void registerPreEmitEPCallback( + const std::function &C) { + PreEmitEPCallbacks.push_back(C); + } + + void registerPostBBSectionsEPCallback( + const std::function &C) { + PostBBSectionsEPCallbacks.push_back(C); + } + + void registerMIEmitEPCallback( + const std::function &C) { + MIEmitEPCallbacks.push_back(C); + } + + void setAddInstSelectorCallback( + const std::function &C) { + AddInstSelectorCallback = C; + } + + void setCodeGenPreparePassesCallback( + const std::function C) { + AddCodeGenPreparePassesCallback = C; + } + + void setRegAllocFastCallback( + const std::function &C) { + AddRegAllocFastCallback = C; + } + + void setRegAllocOptimizedCallback( + const std::function &C) { + AddRegAllocOptimizedCallback = C; + } + ///@}} + + /// Building block callbacks for codegen pipeline. + void addDefaultCodeGenPreparePasses(ModulePassManager &MPM); + Error addDefaultRegAllocFastPasses(MachineFunctionPassManager &MFPM); + Error addDefaultRegAllocOptimizedPasses(MachineFunctionPassManager &MFPM); + + // New pass manager migration methods, don't use them + // outside llvm! + CGPassBuilderOption &getCGPBO() { return CGPBO; } + /// Register a callback for parsing an AliasAnalysis Name to populate /// the given AAManager \p AA void registerParseAACallback( @@ -647,6 +797,28 @@ class PassBuilder { OptimizationLevel Level, ThinOrFullLTOPhase Phase); + void invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM); + void invokeGCLoweringEPCallbacks(FunctionPassManager &FPM); + void invokeISelPrepareEPCallbacks(ModulePassManager &MPM); + void invokeMachineSSAOptimizationEarlyEPCallbacks( + MachineFunctionPassManager &MFPM); + void + invokeMachineSSAOptimizationLastEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePreRegAllocEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePostRegAllocEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePreRegBankSelectEPCallbacks(MachineFunctionPassManager &MFPM); + void + invokePreGlobalInstructionSelectEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePostGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM); + void invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM); + void + invokeMachineLateOptimizationEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePostBBSectionsEPCallbacks(MachineFunctionPassManager &MFPM); + void invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM); + void invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM); + static bool checkParametrizedPassName(StringRef Name, StringRef PassName) { if (!Name.consume_front(PassName)) return false; @@ -711,6 +883,21 @@ class PassBuilder { void addVectorPasses(OptimizationLevel Level, FunctionPassManager &FPM, bool IsFullLTO); + Error addExceptionHandlingPasses(FunctionPassManager &FPM); + + Error addInstructionSelectorPasses(MachineFunctionPassManager &MFPM); + + void addMachineSSAOptimizationPasses(MachineFunctionPassManager &MFPM); + + Error addMachinePasses(ModulePassManager &MPM, FunctionPassManager &FPM, + MachineFunctionPassManager &MFPM); + + Error addRegisterAllocatorPasses(MachineFunctionPassManager &MFPM); + + Error parseRegAllocOption(StringRef Text); + + bool isOptimizedRegAlloc() const; + static std::optional> parsePipelineText(StringRef Text); @@ -779,6 +966,55 @@ class PassBuilder { 2> PipelineEarlySimplificationEPCallbacks; + // CodeGen extension point callbacks + std::function + CustomCodeGenPipelineBuilderCallback; + + SmallVector, 2> + CodeGenIREarlyEPCallbacks; + SmallVector, 2> + GCLoweringEPCallbacks; + SmallVector, 2> + ISelPrepareEPCallbacks; + SmallVector, 2> + MachineSSAOptimizationEarlyEPCallbacks; + SmallVector, 2> + MachineSSAOptimizationLastEPCallbacks; + SmallVector, 2> + PreRegAllocEPCallbacks; + SmallVector, 2> + PostRegAllocEPCallbacks; + SmallVector, 2> + PreRegBankSelectEPCallbacks; + SmallVector, 2> + PreGlobalInstructionSelectEPCallbacks; + SmallVector, 2> + PostGlobalInstructionSelectEPCallbacks; + SmallVector, 2> + ILPOptsEPCallbacks; + SmallVector, 2> + MachineLateOptimizationEPCallbacks; + SmallVector, 2> + PreSched2EPCallbacks; + SmallVector, 2> + PostRewriteEPCallbacks; + SmallVector, 2> + PreEmitEPCallbacks; + SmallVector, 2> + PostBBSectionsEPCallbacks; + SmallVector, 2> + MIEmitEPCallbacks; + + std::function AddCodeGenPreparePassesCallback; + std::function AddInstSelectorCallback; + std::function AddRegAllocFastCallback; + std::function + AddRegAllocOptimizedCallback; + StringSet<> DisabledPasses; + StringMap RegAllocPasses; + // TODO: Add methods in LLVMTargetMachine so we can get rid of it. + SmallVector, 2> ModuleAnalysisRegistrationCallbacks; SmallVector DisableLayoutFSProfileLoader( "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden, cl::desc("Disable MIRProfileLoader before BlockPlacement")); // Specify FSProfile file name. -static cl::opt +cl::opt FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Flow Sensitive profile file name."), cl::Hidden); // Specify Remapping file for FSProfile. -static cl::opt FSRemappingFile( +cl::opt FSRemappingFile( "fs-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden); @@ -500,6 +500,8 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() { SET_BOOLEAN_OPTION(DisableMergeICmps) SET_BOOLEAN_OPTION(DisableLSR) SET_BOOLEAN_OPTION(DisableConstantHoisting) + SET_BOOLEAN_OPTION(DisableReplaceWithVecLib) + SET_BOOLEAN_OPTION(DisableLayoutFSProfileLoader) SET_BOOLEAN_OPTION(DisableCGP) SET_BOOLEAN_OPTION(DisablePartialLibcallInlining) SET_BOOLEAN_OPTION(DisableSelectOptimize) @@ -542,8 +544,7 @@ void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC, }); } -Expected -TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) { +Expected TargetPassConfig::getStartStopInfo() { auto [StartBefore, StartBeforeInstanceNum] = getPassNameAndInstanceNum(StartBeforeOpt); auto [StartAfter, StartAfterInstanceNum] = diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt index 6425f4934b210..f9011cc823508 100644 --- a/llvm/lib/Passes/CMakeLists.txt +++ b/llvm/lib/Passes/CMakeLists.txt @@ -3,6 +3,7 @@ add_llvm_component_library(LLVMPasses OptimizationLevel.cpp PassBuilder.cpp PassBuilderBindings.cpp + PassBuilderCodeGen.cpp PassBuilderPipelines.cpp PassPlugin.cpp StandardInstrumentations.cpp diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index bc6b449d22abe..985480806f581 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -447,9 +447,24 @@ class RequireAllMachineFunctionPropertiesPass PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO, std::optional PGOOpt, PassInstrumentationCallbacks *PIC) - : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) { - if (TM) + : TM(TM), PTO(PTO), PGOOpt(PGOOpt), CGPBO(getCGPassBuilderOption()), + PIC(PIC) { + if (TM) { + if (TM->Options.EnableIPRA) + CGPBO.RequiresCodeGenSCCOrder = true; + AddCodeGenPreparePassesCallback = [&](ModulePassManager &MPM) { + addDefaultCodeGenPreparePasses(MPM); + }; + AddRegAllocFastCallback = [&](MachineFunctionPassManager &MFPM) { + return addDefaultRegAllocFastPasses(MFPM); + }; + AddRegAllocOptimizedCallback = [&](MachineFunctionPassManager &MFPM) { + return addDefaultRegAllocOptimizedPasses(MFPM); + }; + TM->registerPassBuilderCallbacks(*this); + } + if (PIC) { PIC->registerClassToPassNameCallback([this, PIC]() { // MSVC requires this to be captured if it's used inside decltype. diff --git a/llvm/lib/Passes/PassBuilderCodeGen.cpp b/llvm/lib/Passes/PassBuilderCodeGen.cpp new file mode 100644 index 0000000000000..6fef4e3f99b9a --- /dev/null +++ b/llvm/lib/Passes/PassBuilderCodeGen.cpp @@ -0,0 +1,950 @@ +//===- Construction of code generation pass pipelines ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file provides the implementation of the PassBuilder based on our +/// static pass registry as well as related functionality. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/CodeGenPrepare.h" +#include "llvm/CodeGen/DwarfEHPrepare.h" +#include "llvm/CodeGen/ExpandLargeDivRem.h" +#include "llvm/CodeGen/ExpandLargeFpConvert.h" +#include "llvm/CodeGen/ExpandMemCmp.h" +#include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/FinalizeISel.h" +#include "llvm/CodeGen/GCMetadata.h" +#include "llvm/CodeGen/GlobalMergeFunctions.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/LocalStackSlotAllocation.h" +#include "llvm/CodeGen/LowerEmuTLS.h" +#include "llvm/CodeGen/MachineFunctionAnalysis.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/ReplaceWithVeclib.h" +#include "llvm/CodeGen/SafeStack.h" +#include "llvm/CodeGen/SelectOptimize.h" +#include "llvm/CodeGen/ShadowStackGCLowering.h" +#include "llvm/CodeGen/SjLjEHPrepare.h" +#include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/UnreachableBlockElim.h" +#include "llvm/CodeGen/WasmEHPrepare.h" +#include "llvm/CodeGen/WinEHPrepare.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/Passes/CodeGenPassBuilder.h" // Dummy passes only! +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/ObjCARC.h" +#include "llvm/Transforms/Scalar/ConstantHoisting.h" +#include "llvm/Transforms/Scalar/LoopStrengthReduce.h" +#include "llvm/Transforms/Scalar/MergeICmps.h" +#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" +#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Utils/LowerGlobalDtors.h" +#include "llvm/Transforms/Utils/LowerInvoke.h" + +namespace llvm { +extern cl::opt FSRemappingFile; +} + +using namespace llvm; + +void PassBuilder::invokeCodeGenIREarlyEPCallbacks(ModulePassManager &MPM) { + for (auto &C : CodeGenIREarlyEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeGCLoweringEPCallbacks(FunctionPassManager &FPM) { + for (auto &C : GCLoweringEPCallbacks) + C(FPM); +} + +void PassBuilder::invokeISelPrepareEPCallbacks(ModulePassManager &MPM) { + for (auto &C : ISelPrepareEPCallbacks) + C(MPM); +} + +void PassBuilder::invokeMachineSSAOptimizationEarlyEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationEarlyEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineSSAOptimizationLastEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineSSAOptimizationLastEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreRegBankSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreRegBankSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PreGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostGlobalInstructionSelectEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostGlobalInstructionSelectEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeILPOptsEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : ILPOptsEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMachineLateOptimizationEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : MachineLateOptimizationEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokeMIEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : MIEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreEmitEPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreEmitEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostRegAllocEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostRegAllocEPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePreSched2EPCallbacks(MachineFunctionPassManager &MFPM) { + for (auto &C : PreSched2EPCallbacks) + C(MFPM); +} + +void PassBuilder::invokePostBBSectionsEPCallbacks( + MachineFunctionPassManager &MFPM) { + for (auto &C : PostBBSectionsEPCallbacks) + C(MFPM); +} + +void PassBuilder::addDefaultCodeGenPreparePasses(ModulePassManager &MPM) { + FunctionPassManager FPM; + // CodeGen prepare + if (TM->getOptLevel() != CodeGenOptLevel::None && !CGPBO.DisableCGP) + FPM.addPass(CodeGenPreparePass(TM)); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); +} + +Error PassBuilder::addDefaultRegAllocFastPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(PHIEliminationPass()); + MFPM.addPass(TwoAddressInstructionPass()); + if (auto Err = addRegAllocPass(MFPM)) + return Err; + return Error::success(); +} + +Error PassBuilder::addDefaultRegAllocOptimizedPasses( + MachineFunctionPassManager &MFPM) { + MFPM.addPass(DetectDeadLanesPass()); + MFPM.addPass(InitUndefPass()); + MFPM.addPass(ProcessImplicitDefsPass()); + + // LiveVariables currently requires pure SSA form. + // + // FIXME: Once TwoAddressInstruction pass no longer uses kill flags, + // LiveVariables can be removed completely, and LiveIntervals can be + // directly computed. (We still either need to regenerate kill flags after + // regalloc, or preferably fix the scavenger to not depend on them). + // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables. + // When LiveVariables is removed this has to be removed/moved either. + // Explicit addition of UnreachableMachineBlockElim allows stopping before + // or after it with -stop-before/-stop-after. + MFPM.addPass(UnreachableMachineBlockElimPass()); + // FIXME: Some X86 tests failed because of incomplete pipeline. + // MFPM.addPass(RequireAnalysisPass()); + + // Edge splitting is smarter with machine loop info. + MFPM.addPass(RequireAnalysisPass()); + MFPM.addPass(PHIEliminationPass()); + + if (CGPBO.EarlyLiveIntervals) + MFPM.addPass(RequireAnalysisPass()); + + MFPM.addPass(TwoAddressInstructionPass()); + MFPM.addPass(RegisterCoalescerPass()); + + // The machine scheduler may accidentally create disconnected components + // when moving subregister definitions around, avoid this by splitting them + // to separate vregs before. Splitting can also improve reg. allocation + // quality. + MFPM.addPass(RenameIndependentSubregsPass()); + + // PreRA instruction scheduling. + MFPM.addPass(MachineSchedulerPass()); + + if (auto Err = addRegAllocPass(MFPM)) + return Err; + // Finally rewrite virtual registers. + MFPM.addPass(VirtRegRewriterPass()); + + // Regalloc scoring for ML-driven eviction - noop except when learning a new + // eviction policy. + MFPM.addPass(RegAllocScoringPass()); + + return Error::success(); +} + +bool PassBuilder::isOptimizedRegAlloc() const { + return CGPBO.OptimizeRegAlloc.value_or(TM->getOptLevel() != + CodeGenOptLevel::None); +} + +// Find the Profile remapping file name. The internal option takes the +// precedence before getting from TargetMachine. +static std::string getFSRemappingFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSRemappingFile.empty()) + return Options.FSRemappingFile; + const std::optional &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileRemappingFile; +} + +// Find the FSProfile file name. The internal option takes the precedence +// before getting from TargetMachine. +static std::string getFSProfileFile(const TargetMachine *TM, + const CGPassBuilderOption &Options) { + if (!Options.FSProfileFile.empty()) + return Options.FSProfileFile; + const std::optional &PGOOpt = TM->getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileFile; +} + +Error PassBuilder::addExceptionHandlingPasses(FunctionPassManager &FPM) { + const MCAsmInfo *MCAI = TM->getMCAsmInfo(); + if (!MCAI) + return make_error("No MCAsmInfo!", inconvertibleErrorCode()); + switch (MCAI->getExceptionHandlingType()) { + case ExceptionHandling::SjLj: + // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both + // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, + // catch info can get misplaced when a selector ends up more than one block + // removed from the parent invoke(s). This could happen when a landing + // pad is shared by multiple invokes and is also a target of a normal + // edge from elsewhere. + FPM.addPass(SjLjEHPreparePass(TM)); + [[fallthrough]]; + case ExceptionHandling::DwarfCFI: + case ExceptionHandling::ARM: + case ExceptionHandling::AIX: + case ExceptionHandling::ZOS: + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::WinEH: + // We support using both GCC-style and MSVC-style exceptions on Windows, so + // add both preparation passes. Each pass will only actually run if it + // recognizes the personality function. + FPM.addPass(WinEHPreparePass()); + FPM.addPass(DwarfEHPreparePass(TM)); + break; + case ExceptionHandling::Wasm: + // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs + // on catchpads and cleanuppads because it does not outline them into + // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we + // should remove PHIs there. + FPM.addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/true)); + FPM.addPass(WasmEHPreparePass()); + break; + case ExceptionHandling::None: + FPM.addPass(LowerInvokePass()); + + // The lower invoke pass may create unreachable code. Remove it. + FPM.addPass(UnreachableBlockElimPass()); + break; + } + return Error::success(); +} + +Error PassBuilder::addInstructionSelectorPasses( + MachineFunctionPassManager &MFPM) { + CodeGenOptLevel OptLevel = TM->getOptLevel(); + + // Core ISel + // Enable FastISel with -fast-isel, but allow that to be overridden. + TM->setO0WantsFastISel(CGPBO.EnableFastISelOption.value_or(true)); + // Determine an instruction selector. + enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; + SelectorType Selector; + + CGPBO.EnableFastISelOption.value_or(false); + if (CGPBO.EnableFastISelOption.value_or(false)) + Selector = SelectorType::FastISel; + + else if (CGPBO.EnableGlobalISelOption.value_or(false) || + (TM->Options.EnableGlobalISel && + !CGPBO.EnableGlobalISelOption.value_or(false))) + Selector = SelectorType::GlobalISel; + else if (OptLevel == CodeGenOptLevel::None && TM->getO0WantsFastISel()) + Selector = SelectorType::FastISel; + else + Selector = SelectorType::SelectionDAG; + + // Set consistently TM.Options.EnableFastISel and EnableGlobalISel. + if (Selector == SelectorType::FastISel) { + TM->setFastISel(true); + TM->setGlobalISel(false); + } else if (Selector == SelectorType::GlobalISel) { + TM->setFastISel(false); + TM->setGlobalISel(true); + } + + // Add instruction selector passes. + if (Selector == SelectorType::GlobalISel) { + MFPM.addPass(IRTranslatorPass()); + MFPM.addPass(LegalizerPass()); + + // Before running the register bank selector, ask the target if it + // wants to run some passes. + invokePreRegBankSelectEPCallbacks(MFPM); + MFPM.addPass(RegBankSelectPass()); + + invokePreGlobalInstructionSelectEPCallbacks(MFPM); + MFPM.addPass(InstructionSelectPass()); + invokePostGlobalInstructionSelectEPCallbacks(MFPM); + + // Pass to reset the MachineFunction if the ISel failed. + MFPM.addPass(ResetMachineFunctionPass( + TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag, + TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable)); + + // Provide a fallback path when we do not want to abort on + // not-yet-supported input. + if (TM->Options.GlobalISelAbort != GlobalISelAbortMode::Enable) { + if (!AddInstSelectorCallback) + return make_error("No InstSelectorCallback!", + inconvertibleErrorCode()); + AddInstSelectorCallback(MFPM); + } + } else { + if (!AddInstSelectorCallback) + return make_error("No InstSelectorCallback!", + inconvertibleErrorCode()); + AddInstSelectorCallback(MFPM); + } + return Error::success(); +} + +void PassBuilder::addMachineSSAOptimizationPasses( + MachineFunctionPassManager &MFPM) { + // Pre-ra tail duplication. + MFPM.addPass(EarlyTailDuplicatePass()); + + // Optimize PHIs before DCE: removing dead PHI cycles may make more + // instructions dead. + MFPM.addPass(OptimizePHIsPass()); + + // This pass merges large allocas. StackSlotColoring is a different pass + // which merges spill slots. + MFPM.addPass(StackColoringPass()); + + // If the target requests it, assign local variables to stack slots relative + // to one another and simplify frame index references where possible. + MFPM.addPass(LocalStackSlotAllocationPass()); + + // With optimization, dead code should already be eliminated. However + // there is one known exception: lowered code for arguments that are only + // used by tail calls, where the tail calls reuse the incoming stack + // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). + MFPM.addPass(DeadMachineInstructionElimPass()); + + // Allow targets to insert passes that improve instruction level parallelism, + // like if-conversion. Such passes will typically need dominator trees and + // loop info, just like LICM and CSE below. + invokeILPOptsEPCallbacks(MFPM); + + MFPM.addPass(EarlyMachineLICMPass()); + MFPM.addPass(MachineCSEPass()); + MFPM.addPass(MachineSinkingPass()); + MFPM.addPass(PeepholeOptimizerPass()); + // Clean-up the dead code that may have been generated by peephole + // rewriting. + MFPM.addPass(DeadMachineInstructionElimPass()); +} + +static Error setStartStop(ModulePassManager &MPM, + PassInstrumentationCallbacks *PIC) { + auto StartStopInfoOrErr = TargetPassConfig::getStartStopInfo(); + if (!StartStopInfoOrErr) + return StartStopInfoOrErr.takeError(); + auto &SSI = *StartStopInfoOrErr; + + if (SSI.StartPass.empty() && SSI.StopPass.empty()) + return Error::success(); + if (!PIC) { + return make_error("Need PassInstrumentationCallbacks!", + inconvertibleErrorCode()); + } + + static const std::vector SpecialPasses = { + "InvalidateAnalysisPass", "MachineVerifierPass", "PrintMIRPass", + "PrintMIRPreparePass", "RequireAnalysisPass", "VerifierPass"}; + + bool Started = SSI.StartPass.empty(), Stopped = false; + // Return true if pass is skipped. + auto Filter = [&, StartInstanceNum = 0u, + StopInstanceNum = 0u](StringRef Name) mutable { + if (isSpecialPass(Name, SpecialPasses)) + return false; + + bool ShouldDrop = true; + StringRef CurPassName = PIC->getPassNameForClassName(Name); + + // Set instance counters correctly. + if (!SSI.StartPass.empty() && CurPassName == SSI.StartPass) { + ++StartInstanceNum; + if (StartInstanceNum == SSI.StartInstanceNum) + Started = true; + } + if (!SSI.StopPass.empty() && CurPassName == SSI.StopPass) { + ++StopInstanceNum; + if (StopInstanceNum == SSI.StopInstanceNum) + Stopped = true; + } + + // Border case. + const bool AtStartBorder = !SSI.StartPass.empty() && Started && + CurPassName == SSI.StartPass && + StartInstanceNum == SSI.StartInstanceNum; + const bool AtStopBorder = !SSI.StopPass.empty() && Stopped && + CurPassName == SSI.StopPass && + StopInstanceNum == SSI.StopInstanceNum; + if (AtStartBorder) + ShouldDrop = SSI.StartAfter; + if (AtStopBorder) + ShouldDrop = !SSI.StopAfter; + if (!AtStartBorder && !AtStopBorder) + ShouldDrop = !Started || Stopped; + + return ShouldDrop; + }; + + MPM.eraseIf(Filter); + if (!Started) { + return make_error( + "Can't find start pass \"" + SSI.StartPass + "\".", + std::make_error_code(std::errc::invalid_argument)); + } + if (!Stopped && !SSI.StopPass.empty()) { + return make_error( + "Can't find stop pass \"" + SSI.StopPass + "\".", + std::make_error_code(std::errc::invalid_argument)); + } + return Error::success(); +} + +Error PassBuilder::addRegisterAllocatorPasses( + MachineFunctionPassManager &MFPM) { + return isOptimizedRegAlloc() ? AddRegAllocOptimizedCallback(MFPM) + : AddRegAllocFastCallback(MFPM); +} + +Error PassBuilder::addRegAllocPass(MachineFunctionPassManager &MFPM, + StringRef FilterName) { + std::optional FilterFunc = + parseRegAllocFilter(FilterName); + if (!FilterFunc) { + return make_error( + formatv("Unknown register filter name: {0}", FilterName).str(), + std::make_error_code(std::errc::invalid_argument)); + } + + if (RegAllocPasses.contains(FilterName)) { + MFPM.addPass(std::move(RegAllocPasses[FilterName])); + return Error::success(); + } + + // Add default register allocator. + if (isOptimizedRegAlloc()) { + MFPM.addPass(RAGreedyPass()); + } else { + RegAllocFastPassOptions Opts; + Opts.Filter = *FilterFunc; + Opts.FilterName = FilterName; + MFPM.addPass(RegAllocFastPass(Opts)); + } + return Error::success(); +} + +Error PassBuilder::addMachinePasses(ModulePassManager &MPM, + FunctionPassManager &FPM, + MachineFunctionPassManager &MFPM) { + CodeGenOptLevel OptLevel = TM->getOptLevel(); + + // Expand pseudo-instructions emitted by ISel. Don't run the verifier before + // FinalizeISel. + MFPM.addPass(FinalizeISelPass()); + + // Add passes that optimize machine instructions in SSA form. + if (OptLevel != CodeGenOptLevel::None) { + invokeMachineSSAOptimizationEarlyEPCallbacks(MFPM); + addMachineSSAOptimizationPasses(MFPM); + invokeMachineSSAOptimizationLastEPCallbacks(MFPM); + } else { + MFPM.addPass(LocalStackSlotAllocationPass()); + } + + if (TM->Options.EnableIPRA) + MFPM.addPass(RegUsageInfoPropagationPass()); + + // Run pre-ra passes. + invokePreRegAllocEPCallbacks(MFPM); + + if (EnableFSDiscriminator) { + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1)); + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty() && !CGPBO.DisableRAFSProfileLoader) + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::Pass1, nullptr)); + } + + if (auto Err = addRegisterAllocatorPasses(MFPM)) + return Err; + + invokePostRegAllocEPCallbacks(MFPM); + + MFPM.addPass(RemoveRedundantDebugValuesPass()); + MFPM.addPass(FixupStatepointCallerSavedPass()); + + // Insert prolog/epilog code. Eliminate abstract frame index references... + if (OptLevel != CodeGenOptLevel::None) { + MFPM.addPass(PostRAMachineSinkingPass()); + MFPM.addPass(ShrinkWrapPass()); + } + + if (!CGPBO.DisablePrologEpilogInserterPass) + MFPM.addPass(PrologEpilogInserterPass()); + /// Add passes that optimize machine instructions after register allocation. + if (OptLevel != CodeGenOptLevel::None) + invokeMachineLateOptimizationEPCallbacks(MFPM); + + // Expand pseudo instructions before second scheduling pass. + MFPM.addPass(ExpandPostRAPseudosPass()); + + // Run pre-sched2 passes. + invokePreSched2EPCallbacks(MFPM); + + if (CGPBO.EnableImplicitNullChecks) + MFPM.addPass(ImplicitNullChecksPass()); + + // Second pass scheduler. + // Let Target optionally insert this pass by itself at some other + // point. + if (OptLevel != CodeGenOptLevel::None && + !TM->targetSchedulesPostRAScheduling()) { + if (CGPBO.MISchedPostRA) + MFPM.addPass(PostMachineSchedulerPass()); + else + MFPM.addPass(PostRASchedulerPass()); + } + + // GC, replacement for GCMachineCodeAnalysis + MFPM.addPass(GCMachineCodeInsertionPass()); + + // Basic block placement. + if (OptLevel != CodeGenOptLevel::None) { + if (EnableFSDiscriminator) { + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass2)); + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty() && !CGPBO.DisableLayoutFSProfileLoader) + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::Pass2, nullptr)); + } + MFPM.addPass(MachineBlockPlacementPass()); + // Run a separate pass to collect block placement statistics. + if (CGPBO.EnableBlockPlacementStats) + MFPM.addPass(MachineBlockPlacementStatsPass()); + } + + // Insert before XRay Instrumentation. + MFPM.addPass(FEntryInserterPass()); + MFPM.addPass(XRayInstrumentationPass()); + MFPM.addPass(PatchableFunctionPass()); + + invokePreEmitEPCallbacks(MFPM); + + if (TM->Options.EnableIPRA) + // Collect register usage information and produce a register mask of + // clobbered registers, to be used to optimize call sites. + MFPM.addPass(RegUsageInfoCollectorPass()); + + // FIXME: Some backends are incompatible with running the verifier after + // addPreEmitPass. Maybe only pass "false" here for those targets? + MFPM.addPass(FuncletLayoutPass()); + + MFPM.addPass(StackMapLivenessPass()); + MFPM.addPass(LiveDebugValuesPass()); + MFPM.addPass(MachineSanitizerBinaryMetadata()); + + if (TM->Options.EnableMachineOutliner && OptLevel != CodeGenOptLevel::None && + CGPBO.EnableMachineOutliner != RunOutliner::NeverOutline) { + bool RunOnAllFunctions = + (CGPBO.EnableMachineOutliner == RunOutliner::AlwaysOutline); + bool AddOutliner = + RunOnAllFunctions || TM->Options.SupportsDefaultOutlining; + if (AddOutliner) { + FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); + if (CGPBO.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(MachineOutlinerPass(RunOnAllFunctions)); + FPM = FunctionPassManager(); + MFPM = MachineFunctionPassManager(); + } + } + + if (CGPBO.GCEmptyBlocks) + MFPM.addPass(GCEmptyBasicBlocksPass()); + + if (EnableFSDiscriminator) + MFPM.addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast)); + + bool NeedsBBSections = + TM->getBBSectionsType() != llvm::BasicBlockSection::None; + // Machine function splitter uses the basic block sections feature. Both + // cannot be enabled at the same time. We do not apply machine function + // splitter if -basic-block-sections is requested. + if (!NeedsBBSections && (TM->Options.EnableMachineFunctionSplitter || + CGPBO.EnableMachineFunctionSplitter)) { + const std::string ProfileFile = getFSProfileFile(TM, CGPBO); + if (!ProfileFile.empty()) { + if (EnableFSDiscriminator) { + MFPM.addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(TM, CGPBO), + sampleprof::FSDiscriminatorPass::PassLast, nullptr)); + } else { + // Sample profile is given, but FSDiscriminator is not + // enabled, this may result in performance regression. + WithColor::warning() + << "Using AutoFDO without FSDiscriminator for MFS may regress " + "performance.\n"; + } + } + MFPM.addPass(MachineFunctionSplitterPass()); + } + + // We run the BasicBlockSections pass if either we need BB sections or BB + // address map (or both). + if (NeedsBBSections || TM->Options.BBAddrMap) { + if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) + MFPM.addPass(BasicBlockPathCloningPass()); + MFPM.addPass(BasicBlockSectionsPass()); + } + + invokePostBBSectionsEPCallbacks(MFPM); + + if (!CGPBO.DisableCFIFixup && TM->Options.EnableCFIFixup) + MFPM.addPass(CFIFixupPass()); + + MFPM.addPass(StackFrameLayoutAnalysisPass()); + + // Add passes that directly emit MI after all other MI passes. + invokeMIEmitEPCallbacks(MFPM); + + return Error::success(); +} + +Error PassBuilder::buildDefaultCodeGenPipeline(ModulePassManager &TopLevelMPM, + raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, + CodeGenFileType FileType, + MCContext &Ctx) { + if (!TM) + return make_error("Need a TargetMachine instance!", + inconvertibleErrorCode()); + + if (CustomCodeGenPipelineBuilderCallback) { + ModulePassManager MPM; + if (auto Err = CustomCodeGenPipelineBuilderCallback(MPM, Out, DwoOut, + FileType, Ctx)) + return Err; + MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); }); + if (auto Err = setStartStop(MPM, PIC)) + return Err; + TopLevelMPM.addPass(std::move(MPM)); + return Error::success(); + } + + CodeGenOptLevel OptLevel = TM->getOptLevel(); + if (auto Err = parseRegAllocOption(CGPBO.RegAlloc)) + return Err; + + bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline(); + bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null; + + ModulePassManager MPM; + FunctionPassManager FPM; + MachineFunctionPassManager MFPM; + + if (!CGPBO.DisableVerify) + MPM.addPass(VerifierPass()); + + // IR part + // TODO: Remove RequireAnalysisPass() + // when we port AsmPrinter. + MPM.addPass(RequireAnalysisPass()); + MPM.addPass(RequireAnalysisPass()); + MPM.addPass(RequireAnalysisPass()); + + invokeCodeGenIREarlyEPCallbacks(MPM); + + if (TM->useEmulatedTLS()) + MPM.addPass(LowerEmuTLSPass()); + MPM.addPass(PreISelIntrinsicLoweringPass(TM)); + + // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with + // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. + if (TM->getTargetTriple().isOSBinFormatMachO() && + !CGPBO.DisableAtExitBasedGlobalDtorLowering) + MPM.addPass(LowerGlobalDtorsPass()); + + FPM.addPass(ExpandLargeDivRemPass(TM)); + FPM.addPass(ExpandLargeFpConvertPass(TM)); + + // Run loop strength reduction before anything else. + if (OptLevel != CodeGenOptLevel::None) { + if (!CGPBO.DisableLSR) { + LoopPassManager LPM; + LPM.addPass(LoopStrengthReducePass()); + FPM.addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), + /*UseMemorySSA=*/true)); + } + // The MergeICmpsPass tries to create memcmp calls by grouping sequences of + // loads and compares. ExpandMemCmpPass then tries to expand those calls + // into optimally-sized loads and compares. The transforms are enabled by a + // target lowering hook. + if (!CGPBO.DisableMergeICmps) + FPM.addPass(MergeICmpsPass()); + FPM.addPass(ExpandMemCmpPass(TM)); + } + + // Run GC lowering passes for builtin collectors + FPM.addPass(GCLoweringPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(ShadowStackGCLoweringPass()); + FPM = FunctionPassManager(); + invokeGCLoweringEPCallbacks(FPM); + + // Make sure that no unreachable blocks are instruction selected. + FPM.addPass(UnreachableBlockElimPass()); + + if (OptLevel != CodeGenOptLevel::None) { + if (!CGPBO.DisableConstantHoisting) + FPM.addPass(ConstantHoistingPass()); + if (!CGPBO.DisableReplaceWithVecLib) + FPM.addPass(ReplaceWithVeclib()); + if (!CGPBO.DisablePartialLibcallInlining) + FPM.addPass(PartiallyInlineLibCallsPass()); + } + + // Instrument function entry after all inlining. + FPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); + + // Add scalarization of target's unsupported masked memory intrinsics pass. + // the unsupported intrinsic will be replaced with a chain of basic blocks, + // that stores/loads element one-by-one if the appropriate mask bit is set. + FPM.addPass(ScalarizeMaskedMemIntrinPass()); + + // Expand reduction intrinsics into shuffle sequences if the target wants to. + // Allow disabling it for testing purposes. + if (!CGPBO.DisableExpandReductions) + FPM.addPass(ExpandReductionsPass()); + + if (OptLevel != CodeGenOptLevel::None) { + // Convert conditional moves to conditional jumps when profitable. + if (!CGPBO.DisableSelectOptimize) + FPM.addPass(SelectOptimizePass(TM)); + } + + { + ModulePassManager CGMPM; + if (CGPBO.EnableGlobalMergeFunc) + CGMPM.addPass(GlobalMergeFuncPass()); + AddCodeGenPreparePassesCallback(CGMPM); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(std::move(CGMPM)); + FPM = FunctionPassManager(); + } + + // Turn exception handling constructs into something the code generators can + // handle. + if (auto Err = addExceptionHandlingPasses(FPM)) + return Err; + + // All passes after this point need to handle cgscc. + + { // Pre isel extension + ModulePassManager ISelPreparePasses; + invokeISelPrepareEPCallbacks(ISelPreparePasses); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(std::move(ISelPreparePasses)); + } + + if (OptLevel != CodeGenOptLevel::None) + FPM.addPass(ObjCARCContractPass()); + FPM.addPass(CallBrPreparePass()); + // Add both the safe stack and the stack protection passes: each of them will + // only protect functions that have corresponding attributes. + FPM.addPass(SafeStackPass(TM)); + FPM.addPass(StackProtectorPass(TM)); + + // All passes which modify the LLVM IR are now complete; run the verifier + // to ensure that the IR is valid. + if (!CGPBO.DisableVerify) + FPM.addPass(VerifierPass()); + + if (PrintMIR) { + if (CGPBO.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(PrintMIRPreparePass(Out)); + FPM = FunctionPassManager(); + } + + if (auto Err = addInstructionSelectorPasses(MFPM)) + return Err; + + if (auto Err = addMachinePasses(MPM, FPM, MFPM)) + return Err; + + if (!CGPBO.DisableVerify) + MFPM.addPass(MachineVerifierPass()); + + if (PrintMIR) + MFPM.addPass(PrintMIRPass(Out)); + + // TODO: Add AsmPrinter. + (void)Ctx; + + FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))); + FPM.addPass(InvalidateAnalysisPass()); + if (CGPBO.RequiresCodeGenSCCOrder) + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor( + createCGSCCToFunctionPassAdaptor(std::move(FPM)))); + else + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + + MPM.eraseIf([&](StringRef Name) { return DisabledPasses.contains(Name); }); + if (auto Err = setStartStop(MPM, PIC)) + return Err; + TopLevelMPM.addPass(std::move(MPM)); + return Error::success(); +} + +Expected PassBuilder::buildDefaultCodeGenPipeline( + raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, + MCContext &Ctx) { + ModulePassManager MPM; + Error Err = buildDefaultCodeGenPipeline(MPM, Out, DwoOut, FileType, Ctx); + if (Err) + return std::move(Err); + return std::move(MPM); +} + +static bool isRegAllocPass(StringRef Name) { + // TODO: Add all register allocator names. + return Name.starts_with("regallocfast"); +} + +Error PassBuilder::parseRegAllocOption(StringRef Text) { + if (Text == "default") + return Error::success(); + + RegAllocPasses.clear(); + while (!Text.empty()) { + StringRef SinglePass; + std::tie(SinglePass, Text) = Text.split(','); + + if (!isRegAllocPass(SinglePass)) { + return make_error( + formatv("{0} is not a register allocator!", SinglePass).str(), + std::make_error_code(std::errc::invalid_argument)); + } + if (!isOptimizedRegAlloc() && !SinglePass.starts_with("regallocfast")) { + return make_error( + "Must use fast (default) register allocator for unoptimized " + "regalloc.", + std::make_error_code(std::errc::invalid_argument)); + } + + MachineFunctionPassManager MFPM; + if (auto Err = parsePassPipeline(MFPM, SinglePass)) + return Err; + + auto FilterPos = SinglePass.find("filter="); + if (FilterPos == std::string::npos) { + bool Success = RegAllocPasses.try_emplace("all", std::move(MFPM)).second; + if (!Success) { + return make_error( + formatv("Already set register allocator '{0}' for all registers!", + SinglePass) + .str(), + std::make_error_code(std::errc::invalid_argument)); + } + continue; + } + + StringRef FilterName = SinglePass.drop_front(FilterPos); + FilterName.consume_front("filter="); + FilterName = + FilterName.take_until([](char C) { return C == ';' || C == '>'; }); + bool Success = + RegAllocPasses.try_emplace(FilterName, std::move(MFPM)).second; + if (!Success) { + return make_error( + formatv("Already set register allocator '{0}' for filter {1}!", + SinglePass, FilterName) + .str(), + std::make_error_code(std::errc::invalid_argument)); + } + } + return Error::success(); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index da18f2b20f142..427d13ba00c6c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -58,9 +58,11 @@ #include "llvm/CodeGen/MIRParser/MIParser.h" #include "llvm/CodeGen/MachineCSE.h" #include "llvm/CodeGen/MachineLICM.h" +#include "llvm/CodeGen/PHIElimination.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TwoAddressInstructionPass.h" #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PatternMatch.h" @@ -739,6 +741,8 @@ parseAMDGPUAttributorPassOptions(StringRef Params) { } void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { + CGPassBuilderOption &CGPTO = PB.getCGPBO(); + CGPTO.RequiresCodeGenSCCOrder = true; #define GET_PASS_REGISTRY "AMDGPUPassRegistry.def" #include "llvm/Passes/TargetPassRegistry.inc" @@ -849,6 +853,31 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { return onlyAllocateVGPRs; return nullptr; }); + + // CodeGen pass builder part + PB.registerISelPrepareEPCallback([](ModulePassManager &MPM) { + MPM.addPass(createModuleToFunctionPassAdaptor( + RequireAnalysisPass())); + }); + + PB.setAddInstSelectorCallback([&](MachineFunctionPassManager &MFPM) { + MFPM.addPass(AMDGPUISelDAGToDAGPass(*this)); + MFPM.addPass(SIFixSGPRCopiesPass()); + MFPM.addPass(SILowerI1CopiesPass()); + }); + + PB.setRegAllocFastCallback([&](MachineFunctionPassManager &MFPM) -> Error { + // TODO: Add complete pipeline. + MFPM.addPass(PHIEliminationPass()); + // MFPM.addPass(SILowerControlFlowPass()); + MFPM.addPass(TwoAddressInstructionPass()); + // MFPM.addPass(SIWholeQuadModePass()); + if (auto Err = PB.addRegAllocPass(MFPM, "sgpr")) + return Err; + if (auto Err = PB.addRegAllocPass(MFPM, "vgpr")) + return Err; + return Error::success(); + }); } int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) { diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp index d979517e12af6..f911b45b76fcc 100644 --- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp +++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp @@ -53,6 +53,10 @@ Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { void X86TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { #define GET_PASS_REGISTRY "X86PassRegistry.def" #include "llvm/Passes/TargetPassRegistry.inc" + + PB.setAddInstSelectorCallback([this](MachineFunctionPassManager &MFPM) { + MFPM.addPass(X86ISelDAGToDAGPass(*this)); + }); } Error X86TargetMachine::buildCodeGenPipeline( diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-select.ll b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll new file mode 100644 index 0000000000000..7327e1bf95e1e --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/regalloc-select.ll @@ -0,0 +1,9 @@ +; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=DEFAULT +; RUN: llc -mtriple=amdgcn-- -O0 -enable-new-pm -regalloc-npm='regallocfast' -print-pipeline-passes -filetype=null %s | FileCheck %s --check-prefix=CUSTOM + +; DEFAULT: regallocfast +; DEFAULT: regallocfast + +; Just a proof of concept that we can modify parameters of register allocator. +; CUSTOM: regallocfast +; CUSTOM: regallocfast diff --git a/llvm/test/tools/llc/new-pm/start-stop.ll b/llvm/test/tools/llc/new-pm/start-stop.ll index 9c3b9f009178f..8949dd35f67fe 100644 --- a/llvm/test/tools/llc/new-pm/start-stop.ll +++ b/llvm/test/tools/llc/new-pm/start-stop.ll @@ -1,5 +1,5 @@ ; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -filetype=null %s | FileCheck --match-full-lines %s --check-prefix=NULL ; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -o /dev/null %s | FileCheck --match-full-lines %s --check-prefix=OBJ -; NULL: require,require,require,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument,verify) -; OBJ: require,require,require,function(verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,ee-instrument,verify),PrintMIRPreparePass,function(machine-function(print),invalidate) +; NULL: verify,require,require,require,function(mergeicmps,expand-memcmp,gc-lowering),function(verify,machine-function(require,verify),invalidate) +; OBJ: verify,require,require,require,function(mergeicmps,expand-memcmp,gc-lowering),function(verify),PrintMIRPreparePass,function(machine-function(require,verify,print),invalidate) diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp index 3892fbb8c74f7..f40aac39c76b2 100644 --- a/llvm/tools/llc/NewPMDriver.cpp +++ b/llvm/tools/llc/NewPMDriver.cpp @@ -120,6 +120,9 @@ int llvm::compileModuleWithNewPM( CGSCCAnalysisManager CGAM; ModuleAnalysisManager MAM; PassBuilder PB(Target.get(), PipelineTuningOptions(), std::nullopt, &PIC); + CGPassBuilderOption &CGPBO = PB.getCGPBO(); + CGPBO.DisableVerify = VK != VerifierKind::InputOutput; + CGPBO.RegAlloc = RegAlloc; PB.registerModuleAnalyses(MAM); PB.registerCGSCCAnalyses(CGAM); PB.registerFunctionAnalyses(FAM); @@ -156,8 +159,9 @@ int llvm::compileModuleWithNewPM( if (MIR->parseMachineFunctions(*M, MAM)) return 1; } else { - ExitOnErr(Target->buildCodeGenPipeline( - MPM, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC)); + ExitOnErr(PB.buildDefaultCodeGenPipeline(MPM, *OS, + DwoOut ? &DwoOut->os() : nullptr, + FileType, MMI.getContext())); } if (PrintPipelinePasses) {