From f300069baae008ae139c8b20a07ffc69cb903d05 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Wed, 1 Nov 2023 10:41:58 +0800 Subject: [PATCH 1/4] [NewPM][CodeGen] add TargetPassConfig like API --- .../include/llvm/CodeGen/CodeGenPassBuilder.h | 359 ++++++++++++++---- .../include/llvm/CodeGen/MachinePassManager.h | 11 +- llvm/lib/CodeGen/MachinePassManager.cpp | 18 +- 3 files changed, 287 insertions(+), 101 deletions(-) diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index ac9e66c4dc2fd..99e7486c6e7a6 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -19,14 +19,18 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ScopedNoAliasAA.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" +#include "llvm/CodeGen/ExpandLargeDivRem.h" +#include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/ExpandVectorPredication.h" #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/IndirectBrExpand.h" #include "llvm/CodeGen/InterleavedAccess.h" @@ -50,6 +54,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/WithColor.h" #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/CFGuard.h" @@ -60,7 +65,10 @@ #include "llvm/Transforms/Scalar/MergeICmps.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Scalar/TLSVariableHoist.h" +#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" +#include "llvm/Transforms/Utils/LowerGlobalDtors.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include #include @@ -124,6 +132,20 @@ namespace llvm { /// construction. The \c MachinePassRegistry.def file specifies how to construct /// all of the built-in passes, and those may reference these members during /// construction. +/// +/// Target should provide following methods: +/// Parse single target-specific MIR pass +/// @param Name the pass name +/// @return true if failed +/// bool parseTargetMIRPass(MachineFunctionPassManager &MFPM, +/// StringRef Name) const; +/// +/// addPreISel - This method should add any "last minute" LLVM->LLVM +/// passes (which are run just before instruction selector). +/// void addPreISel(AddIRPass &) const; +/// +/// void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; + template class CodeGenPassBuilder { public: explicit CodeGenPassBuilder(LLVMTargetMachine &TM, CGPassBuilderOption Opts, @@ -148,6 +170,28 @@ template class CodeGenPassBuilder { raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const; + /// Parse single non-target-specific MIR pass + /// @param Name the pass name + /// @return true if failed + bool parseMIRPass(MachineFunctionPassManager &MFPM, StringRef Name) const; + + /// Parse MIR pass pipeline. Unlike IR pass pipeline, + /// there is only one pass manager for machine function + /// so there is no need to specify the pass nesting. + /// @param Text a comma separated pass name list + Error parseMIRPipeline(MachineFunctionPassManager &MFPM, + StringRef Text) const { + for (auto [LHS, RHS] = Text.split(','); LHS != ""; + std::tie(LHS, RHS) = RHS.split(',')) { + if (parseMIRPass(MFPM, LHS) && derived().parseTargetMIRPass(MFPM, LHS)) { + return createStringError( + std::make_error_code(std::errc::invalid_argument), + Twine('\"') + Twine(LHS) + Twine("\" pass could not be found.")); + } + } + return Error::success(); + } + void registerModuleAnalyses(ModuleAnalysisManager &) const; void registerFunctionAnalyses(FunctionAnalysisManager &) const; void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const; @@ -160,12 +204,18 @@ template class CodeGenPassBuilder { } PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { - return PIC; + static PassInstrumentationCallbacks PseudoPIC; + return PIC ? PIC : &PseudoPIC; } -protected: - template using has_key_t = decltype(PassT::Key); + /// Allow the target to disable a specific standard pass by default. + template void disablePass() { + DisabledPasses.insert(PassT::ID()); + getPassInstrumentationCallbacks()->registerShouldRunOptionalPassCallback( + [](StringRef P, Any IR) { return P != PassT::name(); }); + } +protected: template using is_module_pass_t = decltype(std::declval().run( std::declval(), std::declval())); @@ -224,38 +274,57 @@ template class CodeGenPassBuilder { AddMachinePass(MachineFunctionPassManager &PM) : PM(PM) {} template void operator()(PassT &&Pass) { - static_assert( - is_detected::value, - "Machine function pass must define a static member variable `Key`."); for (auto &C : BeforeCallbacks) - if (!C(&PassT::Key)) + if (!C(PassT::ID())) return; PM.addPass(std::forward(Pass)); for (auto &C : AfterCallbacks) - C(&PassT::Key); - } - - template void insertPass(AnalysisKey *ID, PassT Pass) { - AfterCallbacks.emplace_back( - [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) { - if (PassID == ID) - this->PM.addPass(std::move(Pass)); - }); + C(PassT::ID(), PassT::name()); } - void disablePass(AnalysisKey *ID) { - BeforeCallbacks.emplace_back( - [ID](AnalysisKey *PassID) { return PassID != ID; }); + template void insertPass(MachinePassKey *ID, PassT Pass) { + AfterCallbacks.emplace_back([this, ID, Pass = std::move(Pass)]( + MachinePassKey *PassID, StringRef) { + if (PassID == ID) + this->PM.addPass(std::move(Pass)); + }); } MachineFunctionPassManager releasePM() { return std::move(PM); } private: MachineFunctionPassManager &PM; - SmallVector, 4> BeforeCallbacks; - SmallVector, 4> AfterCallbacks; + SmallVector, 4> + BeforeCallbacks; + SmallVector, 4> + AfterCallbacks; }; + // Find the FSProfile file name. The internal option takes the precedence + // before getting from TargetMachine. + // TODO: Use PGOOptions only. + std::string getFSProfileFile() const { + if (!Opt.FSProfileFile.empty()) + return Opt.FSProfileFile; + const std::optional &PGOOpt = TM.getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileFile; + } + + // Find the Profile remapping file name. The internal option takes the + // precedence before getting from TargetMachine. + // TODO: Use PGOOptions only. + std::string getFSRemappingFile() const { + if (!Opt.FSRemappingFile.empty()) + return Opt.FSRemappingFile; + const std::optional &PGOOpt = TM.getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileRemappingFile; + } + + DenseSet DisabledPasses; LLVMTargetMachine &TM; CGPassBuilderOption Opt; PassInstrumentationCallbacks *PIC; @@ -264,9 +333,6 @@ template class CodeGenPassBuilder { void registerTargetAnalysis(ModuleAnalysisManager &) const {} void registerTargetAnalysis(FunctionAnalysisManager &) const {} void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {} - std::pair getTargetPassNameFromLegacyName(StringRef) const { - return {"", false}; - } template TMC &getTM() const { return static_cast(TM); } CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } @@ -334,12 +400,14 @@ template class CodeGenPassBuilder { /// immediately before machine code is emitted. void addPreEmitPass(AddMachinePass &) const {} + /// This pass may be implemented by targets that want to run passes + /// immediately after basic block sections are assigned. + void addPostBBSections(AddMachinePass &) const {} + /// Targets may add passes immediately before machine code is emitted in this /// callback. This is called even later than `addPreEmitPass`. - // FIXME: Rename `addPreEmitPass` to something more sensible given its actual - // position and remove the `2` suffix here as this callback is what - // `addPreEmitPass` *should* be but in reality isn't. - void addPreEmitPass2(AddMachinePass &) const {} + /// This function replaces `addPreEmitPass2` in TargetConfig. + void addPrecedingEmitPass(AddMachinePass &) const {} /// {{@ For GlobalISel /// @@ -399,7 +467,7 @@ template class CodeGenPassBuilder { /// representation to the MI representation. /// Adds IR based lowering and target specific optimization passes and finally /// the core instruction selection passes. - void addISelPasses(AddIRPass &) const; + Error addISelPasses(AddIRPass &, AddMachinePass &) const; /// Add the actual instruction selection passes. This does not include /// preparation passes on IR. @@ -439,6 +507,14 @@ template class CodeGenPassBuilder { /// are required for fast register allocation. Error addFastRegAlloc(AddMachinePass &) const; + /// addPostFastRegAllocRewrite - Add passes to the optimized register + /// allocation pipeline after fast register allocation is complete. + Error addPostFastRegAllocRewrite(AddMachinePass &) const { + return make_error( + "addPostFastRegAllocRewrite is not overridden", + inconvertibleErrorCode()); + } + /// addOptimizedRegAlloc - Add passes related to register allocation. /// LLVMTargetMachine provides standard regalloc passes for most targets. void addOptimizedRegAlloc(AddMachinePass &) const; @@ -446,14 +522,43 @@ template class CodeGenPassBuilder { /// Add passes that optimize machine instructions after register allocation. void addMachineLateOptimization(AddMachinePass &) const; - /// addGCPasses - Add late codegen passes that analyze code for garbage + /// registerGCPasses - Add late codegen passes that analyze code for garbage /// collection. This should return true if GC info should be printed after /// these passes. - void addGCPasses(AddMachinePass &) const {} + bool registerGCPasses(MachineFunctionAnalysisManager &MFAM) const { + MFAM.registerPass([] { return GCMachineCodeAnalysisPass(); }); + return true; + } /// Add standard basic block placement passes. void addBlockPlacement(AddMachinePass &) const; + /// Add a pass to print the machine function if printing is enabled. + void addPrintPass(AddMachinePass &addPass, const std::string &Banner) const { + if (Opt.PrintAfterISel) + addPass(MachineFunctionPrinterPass(dbgs(), Banner)); + } + + /// Add a pass to perform basic verification of the machine function if + /// verification is enabled. + void addVerifyPass(AddMachinePass &addPass, const std::string &Banner) const { + bool Verify = Opt.VerifyMachineCode.value_or(false); +#ifdef EXPENSIVE_CHECKS + if (!Opt.VerifyMachineCode) + Verify = TM->isMachineVerifierClean(); +#endif + if (Verify) + addPass(MachineVerifierPass(Banner)); + } + + /// printAndVerify - Add a pass to dump then verify the machine function, if + /// those steps are enabled. + void printAndVerify(AddMachinePass &addPass, + const std::string &Banner) const { + addPrintPass(addPass, Banner); + addVerifyPass(addPass, Banner); + } + using CreateMCStreamer = std::function>(MCContext &)>; void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const { @@ -471,10 +576,10 @@ template class CodeGenPassBuilder { /// regalloc pass. void addRegAllocPass(AddMachinePass &, bool Optimized) const; - /// Add core register alloator passes which do the actual register assignment - /// and rewriting. \returns true if any passes were added. - Error addRegAssignmentFast(AddMachinePass &) const; - Error addRegAssignmentOptimized(AddMachinePass &) const; + /// Add core register allocator passes which do the actual register assignment + /// and rewriting. \returns Error::success() if any passes were added. + Error addRegAssignAndRewriteFast(AddMachinePass &addPass) const; + Error addRegAssignAndRewriteOptimized(AddMachinePass &addPass) const; private: DerivedT &derived() { return static_cast(*this); } @@ -491,10 +596,8 @@ Error CodeGenPassBuilder::buildPipeline( AddIRPass addIRPass(MPM, Opt.DebugPM); // `ProfileSummaryInfo` is always valid. addIRPass(RequireAnalysisPass()); - addISelPasses(addIRPass); - AddMachinePass addPass(MFPM); - if (auto Err = addCoreISelPasses(addPass)) + if (auto Err = addISelPasses(addIRPass, addPass)) return std::move(Err); if (auto Err = derived().addMachinePasses(addPass)) @@ -533,12 +636,14 @@ void CodeGenPassBuilder::registerModuleAnalyses( MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; }); #include "MachinePassRegistry.def" derived().registerTargetAnalysis(MAM); + // TODO: add SCC order codegen } template void CodeGenPassBuilder::registerFunctionAnalyses( FunctionAnalysisManager &FAM) const { - FAM.registerPass([this] { return registerAAAnalyses(); }); + if (getOptLevel() != CodeGenOptLevel::None) + FAM.registerPass([this] { return registerAAAnalyses(); }); #define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \ FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; }); @@ -602,17 +707,45 @@ CodeGenPassBuilder::getPassNameFromLegacyName(StringRef Name) const { return Ret; } +template +bool CodeGenPassBuilder::parseMIRPass( + MachineFunctionPassManager &MFPM, StringRef Name) const { +#define ADD_PASS(NAME, PASS_NAME) \ + if (Name == NAME) { \ + MFPM.addPass(PASS_NAME()); \ + return false; \ + } +#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME) +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME) +#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME) +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME) +#include "MachinePassRegistry.def" +#undef ADD_PASS + return true; +} + template -void CodeGenPassBuilder::addISelPasses(AddIRPass &addPass) const { +Error CodeGenPassBuilder::addISelPasses( + AddIRPass &addPass, AddMachinePass &addMachinePass) const { if (TM.useEmulatedTLS()) addPass(LowerEmuTLSPass()); addPass(PreISelIntrinsicLoweringPass(TM)); + addPass(createModuleToFunctionPassAdaptor(ExpandLargeDivRemPass(&TM))); + addPass(createModuleToFunctionPassAdaptor(ExpandLargeFpConvertPass(&TM))); derived().addIRPasses(addPass); derived().addCodeGenPrepare(addPass); addPassesToHandleExceptions(addPass); derived().addISelPrepare(addPass); + + if (auto Err = addCoreISelPasses(addMachinePass)) + return std::move(Err); + return Error::success(); } /// Add common target configurable passes that perform LLVM IR to IR transforms @@ -624,16 +757,15 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { if (!Opt.DisableVerify) addPass(VerifierPass()); - // Run loop strength reduction before anything else. - if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) { - addPass(createFunctionToLoopPassAdaptor( - LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM)); - // FIXME: use -stop-after so we could remove PrintLSR - if (Opt.PrintLSR) - addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n")); - } - if (getOptLevel() != CodeGenOptLevel::None) { + // Run loop strength reduction before anything else. + if (!Opt.DisableLSR) { + addPass(createFunctionToLoopPassAdaptor( + CanonicalizeFreezeInLoopsPass(), /*UseMemorySSA*/ true, Opt.DebugPM)); + addPass(createFunctionToLoopPassAdaptor( + LoopStrengthReducePass(), /*UseMemorySSA*/ true, Opt.DebugPM)); + } + // The MergeICmpsPass tries to create memcmp calls by grouping sequences of // loads and compares. ExpandMemCmpPass then tries to expand those calls // into optimally-sized loads and compares. The transforms are enabled by a @@ -649,6 +781,12 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { addPass(ShadowStackGCLoweringPass()); addPass(LowerConstantIntrinsicsPass()); + // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with + // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. + if (TM.getTargetTriple().isOSBinFormatMachO() && + !Opt.DisableAtExitBasedGlobalDtorLowering) + addPass(LowerGlobalDtorsPass()); + // Make sure that no unreachable blocks are instruction selected. addPass(UnreachableBlockElimPass()); @@ -665,8 +803,10 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { !Opt.DisablePartialLibcallInlining) addPass(PartiallyInlineLibCallsPass()); - // Instrument function entry and exit, e.g. with calls to mcount(). - addPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); + // Expand vector predication intrinsics into standard IR instructions. + // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction + // passes since it emits those kinds of intrinsics. + addPass(ExpandVectorPredicationPass()); // Add scalarization of target's unsupported masked memory intrinsics pass. // the unsupported intrinsic will be replaced with a chain of basic blocks, @@ -674,7 +814,12 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { addPass(ScalarizeMaskedMemIntrinPass()); // Expand reduction intrinsics into shuffle sequences if the target wants to. - addPass(ExpandReductionsPass()); + // Allow disabling it for testing purposes. + if (!Opt.DisableExpandReductions) + addPass(ExpandReductionsPass()); + + if (getOptLevel() != CodeGenOptLevel::None) + addPass(TLSVariableHoistPass()); // Convert conditional moves to conditional jumps when profitable. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) @@ -733,8 +878,6 @@ template void CodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) addPass(CodeGenPreparePass()); - // TODO: Default ctor'd RewriteSymbolPass is no-op. - // addPass(RewriteSymbolPass()); } /// Add common passes that perform LLVM IR to IR transforms in preparation for @@ -831,8 +974,7 @@ Error CodeGenPassBuilder::addCoreISelPasses( addPass(FinalizeISelPass()); // // Print the instruction selected machine code... - // printAndVerify("After Instruction Selection"); - + printAndVerify(addPass, "After Instruction Selection"); return Error::success(); } @@ -869,9 +1011,22 @@ Error CodeGenPassBuilder::addMachinePasses( // Run pre-ra passes. derived().addPreRegAlloc(addPass); + // Add a FSDiscriminator pass right before RA, so that we could get + // more precise SampleFDO profile for RA. + if (EnableFSDiscriminator) { + addPass(MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1)); + const std::string ProfileFile = getFSProfileFile(); + if (!ProfileFile.empty() && !Opt.DisableRAFSProfileLoader) + addPass(MIRProfileLoaderNewPass(ProfileFile, getFSRemappingFile(), + sampleprof::FSDiscriminatorPass::Pass1, + nullptr)); + } + // Run register allocation and passes that are tightly coupled with it, // including phi elimination and scheduling. - if (*Opt.OptimizeRegAlloc) { + bool IsOptimizeRegAlloc = + Opt.OptimizeRegAlloc.value_or(getOptLevel() != CodeGenOptLevel::None); + if (IsOptimizeRegAlloc) { derived().addOptimizedRegAlloc(addPass); } else { if (auto Err = derived().addFastRegAlloc(addPass)) @@ -883,13 +1038,19 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(RemoveRedundantDebugValuesPass()); - // Insert prolog/epilog code. Eliminate abstract frame index references... + addPass(FixupStatepointCallerSavedPass()); + + // Insert prolog/epilog code. Eliminate abstract frame index + // references... if (getOptLevel() != CodeGenOptLevel::None) { addPass(PostRAMachineSinkingPass()); addPass(ShrinkWrapPass()); } - addPass(PrologEpilogInserterPass()); + // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only + // do so if it hasn't been disabled, substituted, or overridden. + if (!DisabledPasses.contains(PrologEpilogCodeInserterPass::ID())) + addPass(PrologEpilogInserterPass()); /// Add passes that optimize machine instructions after register allocation. if (getOptLevel() != CodeGenOptLevel::None) @@ -915,9 +1076,6 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(PostRASchedulerPass()); } - // GC - derived().addGCPasses(addPass); - // Basic block placement. if (getOptLevel() != CodeGenOptLevel::None) derived().addBlockPlacement(addPass); @@ -935,6 +1093,8 @@ Error CodeGenPassBuilder::addMachinePasses( // clobbered registers, to be used to optimize call sites. addPass(RegUsageInfoCollectorPass()); + // FIXME: Some backends are incompatible with running the verifier after + // addPreEmitPass. Maybe only pass "false" here for those targets? addPass(FuncletLayoutPass()); addPass(StackMapLivenessPass()); @@ -951,8 +1111,51 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(MachineOutlinerPass(RunOnAllFunctions)); } + if (Opt.GCEmptyBlocks) + addPass(GCEmptyBasicBlocksPass()); + + if (EnableFSDiscriminator) + addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast)); + + // Machine function splitter uses the basic block sections feature. Both + // cannot be enabled at the same time. Basic block sections takes precedence. + // FIXME: In principle, BasicBlockSection::Labels and splitting can used + // together. Update this check once we have addressed any issues. + if (TM.getBBSectionsType() != llvm::BasicBlockSection::None) { + if (TM.getBBSectionsType() == llvm::BasicBlockSection::List) { + addPass( + BasicBlockSectionsProfileReaderPass(TM.getBBSectionsFuncListBuf())); + } + addPass(BasicBlockSectionsPass()); + } else if (TM.Options.EnableMachineFunctionSplitter || + Opt.EnableMachineFunctionSplitter) { + const std::string ProfileFile = getFSProfileFile(); + if (!ProfileFile.empty()) { + if (EnableFSDiscriminator) { + addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(), + sampleprof::FSDiscriminatorPass::PassLast, nullptr)); + } else { + // Sample profile is given, but FSDiscriminator is not + // enabled, this may result in performance regression. + WithColor::warning() + << "Using AutoFDO without FSDiscriminator for MFS may regress " + "performance."; + } + } + addPass(MachineFunctionSplitterPass()); + } + + derived().addPostBBSections(addPass); + + if (!Opt.DisableCFIFixup && TM.Options.EnableCFIFixup) + addPass(CFIFixupPass()); + + addPass(StackFrameLayoutAnalysisPass()); + // Add passes that directly emit MI after all other MI passes. - derived().addPreEmitPass2(addPass); + derived().addPrecedingEmitPass(addPass); return Error::success(); } @@ -1041,7 +1244,7 @@ void CodeGenPassBuilder::addRegAllocPass(AddMachinePass &addPass, } template -Error CodeGenPassBuilder::addRegAssignmentFast( +Error CodeGenPassBuilder::addRegAssignAndRewriteFast( AddMachinePass &addPass) const { if (Opt.RegAlloc != RegAllocType::Default && Opt.RegAlloc != RegAllocType::Fast) @@ -1049,26 +1252,27 @@ Error CodeGenPassBuilder::addRegAssignmentFast( "Must use fast (default) register allocator for unoptimized regalloc.", inconvertibleErrorCode()); - addRegAllocPass(addPass, false); - return Error::success(); + addPass(RegAllocPass(false)); + + // Allow targets to change the register assignments after + // fast register allocation. + return derived().addPostFastRegAllocRewrite(addPass); } -template -Error CodeGenPassBuilder::addRegAssignmentOptimized( +template +Error CodeGenPassBuilder::addRegAssignAndRewriteOptimized( AddMachinePass &addPass) const { // Add the selected register allocation pass. addRegAllocPass(addPass, true); - // Allow targets to change the register assignments before rewriting. - derived().addPreRewrite(addPass); + addPreRewrite(addPass); // Finally rewrite virtual registers. addPass(VirtRegRewriterPass()); - // Perform stack slot coloring and post-ra machine LICM. - // - // FIXME: Re-enable coloring with register when it's capable of adding - // kill markers. - addPass(StackSlotColoringPass()); + + // Regalloc scoring for ML-driven eviction - noop except when learning a new + // eviction policy. + addPass(RegAllocScoringPass()); return Error::success(); } @@ -1080,7 +1284,7 @@ Error CodeGenPassBuilder::addFastRegAlloc( AddMachinePass &addPass) const { addPass(PHIEliminationPass()); addPass(TwoAddressInstructionPass()); - return derived().addRegAssignmentFast(addPass); + return derived().addRegAssignAndRewriteFast(addPass); } /// Add standard target-independent passes that are tightly coupled with @@ -1111,7 +1315,8 @@ void CodeGenPassBuilder::addOptimizedRegAlloc( // PreRA instruction scheduling. addPass(MachineSchedulerPass()); - if (derived().addRegAssignmentOptimized(addPass)) { + Error Err = derived().addRegAssignAndRewriteOptimized(addPass); + if (!Err) { // Allow targets to expand pseudo instructions depending on the choice of // registers before MachineCopyPropagation. derived().addPostRewrite(addPass); diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index a2641a8223646..c55b7ed157b4e 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -27,6 +27,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Error.h" +#include "llvm/Target/CGPassBuilderOption.h" #include @@ -150,10 +151,7 @@ class MachineFunctionPassManager using Base = PassManager; public: - MachineFunctionPassManager(bool RequireCodeGenSCCOrder = false, - bool VerifyMachineFunction = false) - : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder), - VerifyMachineFunction(VerifyMachineFunction) {} + MachineFunctionPassManager() : Opt(getCGPassBuilderOption()) {} MachineFunctionPassManager(MachineFunctionPassManager &&) = default; MachineFunctionPassManager & operator=(MachineFunctionPassManager &&) = default; @@ -261,10 +259,7 @@ class MachineFunctionPassManager using PassIndex = decltype(Passes)::size_type; std::map> MachineModulePasses; - // Run codegen in the SCC order. - bool RequireCodeGenSCCOrder; - - bool VerifyMachineFunction; + CGPassBuilderOption Opt; }; } // end namespace llvm diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp index 914e6b19fde9a..52722cad0c2ce 100644 --- a/llvm/lib/CodeGen/MachinePassManager.cpp +++ b/llvm/lib/CodeGen/MachinePassManager.cpp @@ -22,6 +22,7 @@ template class AllAnalysesOn; template class AnalysisManager; template class PassManager; +// TODO: Add a way to run verifier and debugify passes. Error MachineFunctionPassManager::run(Module &M, MachineFunctionAnalysisManager &MFAM) { // MachineModuleAnalysis is a module analysis pass that is never invalidated @@ -30,26 +31,11 @@ Error MachineFunctionPassManager::run(Module &M, // result of MachineModuleAnalysis. MMI should not be recomputed. auto &MMI = MFAM.getResult(M); - (void)RequireCodeGenSCCOrder; - assert(!RequireCodeGenSCCOrder && "not implemented"); + assert(!Opt.RequiresCodeGenSCCOrder && "not implemented"); // M is unused here PassInstrumentation PI = MFAM.getResult(M); - // Add a PIC to verify machine functions. - if (VerifyMachineFunction) { - // No need to pop this callback later since MIR pipeline is flat which means - // current pipeline is the top-level pipeline. Callbacks are not used after - // current pipeline. - PI.pushBeforeNonSkippedPassCallback([&MFAM](StringRef PassID, Any IR) { - assert(llvm::any_cast(&IR)); - const MachineFunction *MF = llvm::any_cast(IR); - assert(MF && "Machine function should be valid for printing"); - std::string Banner = std::string("After ") + std::string(PassID); - verifyMachineFunction(&MFAM, Banner, *MF); - }); - } - for (auto &F : InitializationFuncs) { if (auto Err = F(M, MFAM)) return Err; From c2aae6d9699915dd1c95d268f024f3673fbcbb9d Mon Sep 17 00:00:00 2001 From: Yuanfang Chen <455423+yuanfang-chen@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:44:15 +0800 Subject: [PATCH 2/4] [X86][CodeGen] Add NPM pipeline builder --- llvm/lib/Target/X86/CMakeLists.txt | 2 + llvm/lib/Target/X86/X86PassRegistry.def | 76 +++++ llvm/lib/Target/X86/X86TargetMachine.cpp | 358 ++++++++++++++++++++++- llvm/lib/Target/X86/X86TargetMachine.h | 12 + 4 files changed, 442 insertions(+), 6 deletions(-) create mode 100644 llvm/lib/Target/X86/X86PassRegistry.def diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 0b7a98ad6341d..ef54ca3e435c5 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -99,8 +99,10 @@ add_llvm_target(X86CodeGen ${sources} Core GlobalISel Instrumentation + IRPrinter MC ProfileData + ScalarOpts SelectionDAG Support Target diff --git a/llvm/lib/Target/X86/X86PassRegistry.def b/llvm/lib/Target/X86/X86PassRegistry.def new file mode 100644 index 0000000000000..b591fc8ccd0d4 --- /dev/null +++ b/llvm/lib/Target/X86/X86PassRegistry.def @@ -0,0 +1,76 @@ +//===- X86PassRegistry.def - Registry of passes for X86 ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// X86 pass registry +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef FUNCTION_PASS + +#ifndef DUMMY_FUNCTION_PASS +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_FUNCTION_PASS("x86-win-eh-state", X86WinEHStatePass, ()) +#undef DUMMY_FUNCTION_PASS + +#ifndef MACHINE_FUNCTION_PASS +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef MACHINE_FUNCTION_PASS + +// PASS_NAME is for mocking machine passes, remove it after all machine passes +// are added new pass manager interface. +#ifndef DUMMY_MACHINE_FUNCTION_PASS +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_MACHINE_FUNCTION_PASS("x86-isel-dag", X86ISelDagPass, (getTM(), getOptLevel())) +DUMMY_MACHINE_FUNCTION_PASS("x86-global-basereg", X86GlobalBaseRegPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-cmov-converter", X86CmovConverterDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-setcc", X86FixupSetCCPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-opt-leas", X86OptimizeLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-callframe-opt", X86CallFrameOptimizationPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-store-foword-block", X86AvoidStoreForwardingBlocksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-speculative-load-hardening", X86SpeculativeLoadHardeningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-flags-copy-lowering", X86FlagsCopyLoweringDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-domain-reassign", X86DomainReassignmentPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fp-stackifier", X86FloatingPointStackifierPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-expand-pseudo", X86ExpandPseudoPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-exec-domain-fix", X86ExecutionDomainFixPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-indirectbr-tracking", X86IndirectBranchTrackingPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-issue-vzero-upper", X86IssueVZeroUpperPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-bwinsts", X86FixupBWInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-pad-short-funcs", X86PadShortFunctionsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-leas", X86FixupLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-evex-to-vex-insts", X86EvexToVexInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-discriminate-memops", X86DiscriminateMemOpsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-prefetch", X86InsertPrefetchPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-x87-wait", X86InsertX87waitPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-trailing-call", X86AvoidTrailingCallPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-asm-printer", X86AsmPrinterPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("cleanup-local-dyn-tls", CleanupLocalDynamicTLSPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-inst-tuning", X86FixupInstTuningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-vector-constants", X86FixupVectorConstantsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("ehcontguard-catchret", EHContGuardCatchretPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-lvi-ret", X86LoadValueInjectionRetHardeningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-lower-tile-copy", X86LowerTileCopyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-seses", X86SpeculativeExecutionSideEffectSuppressionPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-retpoline-thunks", X86IndirectThunksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-return-thunks", X86ReturnThunksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("tileconfig", X86TileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("tile-pre-config", X86PreTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("fastpretileconfig", X86FastPreTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("fasttileconfig", X86FastTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-dyn-alloca-expander", X86DynAllocaExpanderPass, ()) +#undef DUMMY_MACHINE_FUNCTION_PASS diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 5668b514d6dec..9de8724cc30f8 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CodeGenPassBuilder.h" #include "llvm/CodeGen/ExecutionDomainFix.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" @@ -39,6 +40,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" @@ -48,15 +50,17 @@ #include "llvm/Target/TargetOptions.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/CFGuard.h" +#include "llvm/Transforms/Instrumentation/KCFI.h" #include #include #include using namespace llvm; -static cl::opt EnableMachineCombinerPass("x86-machine-combiner", - cl::desc("Enable the machine combiner pass"), - cl::init(true), cl::Hidden); +static cl::opt + EnableMachineCombinerPass("x86-machine-combiner", + cl::desc("Enable the machine combiner pass"), + cl::init(true), cl::Hidden); static cl::opt EnableTileRAPass("x86-tile-ra", @@ -365,7 +369,7 @@ namespace { class X86PassConfig : public TargetPassConfig { public: X86PassConfig(X86TargetMachine &TM, PassManagerBase &PM) - : TargetPassConfig(TM, PM) {} + : TargetPassConfig(TM, PM) {} X86TargetMachine &getX86TargetMachine() const { return getTM(); @@ -418,10 +422,10 @@ char X86ExecutionDomainFix::ID; } // end anonymous namespace INITIALIZE_PASS_BEGIN(X86ExecutionDomainFix, "x86-execution-domain-fix", - "X86 Execution Domain Fix", false, false) + "X86 Execution Domain Fix", false, false) INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) INITIALIZE_PASS_END(X86ExecutionDomainFix, "x86-execution-domain-fix", - "X86 Execution Domain Fix", false, false) + "X86 Execution Domain Fix", false, false) TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) { return new X86PassConfig(*this, PM); @@ -659,3 +663,345 @@ bool X86PassConfig::addRegAssignAndRewriteOptimized() { } return TargetPassConfig::addRegAssignAndRewriteOptimized(); } + +namespace { + +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public PassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + }; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public MachinePassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(MachineFunction &, \ + MachineFunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + static MachinePassKey Key; \ + }; \ + MachinePassKey PASS_NAME::Key; +#include "X86PassRegistry.def" + +/// X86 Code Generator Pass Configuration Options. +struct X86CodeGenPassBuilder + : public CodeGenPassBuilder { + X86CodeGenPassBuilder(X86TargetMachine &TM, + CGPassBuilderOption Opt = CGPassBuilderOption(), + PassInstrumentationCallbacks *PIC = nullptr) + : CodeGenPassBuilder(TM, Opt, PIC) { + // Target-specific `CGPassBuilderOption` could be overridden here. + } + + std::pair getTargetPassNameFromLegacyName(StringRef) const; + + bool parseTargetMIRPass(MachineFunctionPassManager &MFPM, + StringRef Text) const; + void addIRPasses(AddIRPass &) const; + void addPreISel(AddIRPass &) const; + Error addInstSelector(AddMachinePass &) const; + Error addIRTranslator(AddMachinePass &) const; + Error addLegalizeMachineIR(AddMachinePass &) const; + Error addRegBankSelect(AddMachinePass &) const; + Error addGlobalInstructionSelect(AddMachinePass &) const; + void addILPOpts(AddMachinePass &) const; + void addMachineSSAOptimization(AddMachinePass &) const; + void addPreRegAlloc(AddMachinePass &) const; + Error addPostFastRegAllocRewrite(AddMachinePass &) const; + void addPostRegAlloc(AddMachinePass &) const; + void addPreEmitPass(AddMachinePass &) const; + void addPrecedingEmitPass(AddMachinePass &) const; + void addPreSched2(AddMachinePass &) const; + Error addRegAssignAndRewriteOptimized(AddMachinePass &) const; + void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; +}; + +} // namespace + +void X86CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { + addPass(AtomicExpandPass()); + + CodeGenPassBuilder::addIRPasses(addPass); + + if (TM.getOptLevel() != CodeGenOptLevel::None) + addPass(InterleavedAccessPass(&TM)); + + // Add passes that handle indirect branch removal and insertion of a retpoline + // thunk. These will be a no-op unless a function subtarget has the retpoline + // feature enabled. + addPass(IndirectBrExpandPass(&TM)); + + // Add Control Flow Guard checks. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows()) { + if (TT.getArch() == Triple::x86_64) + addPass(CFGuardPass(CFGuardPass::Mechanism::Dispatch)); + else + addPass(CFGuardPass(CFGuardPass::Mechanism::Check)); + } +} + +Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { + // Install an instruction selector. + addPass(X86ISelDagPass(getTM(), getOptLevel())); + + // For ELF, cleanup any local-dynamic TLS accesses. + if (TM.getTargetTriple().isOSBinFormatELF() && + TM.getOptLevel() != CodeGenOptLevel::None) + addPass(CleanupLocalDynamicTLSPass()); + + addPass(X86GlobalBaseRegPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addIRTranslator(AddMachinePass &addPass) const { + addPass(IRTranslatorPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addLegalizeMachineIR( + AddMachinePass &addPass) const { + addPass(LegalizerPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addRegBankSelect(AddMachinePass &addPass) const { + addPass(RegBankSelectPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addGlobalInstructionSelect( + AddMachinePass &addPass) const { + addPass(InstructionSelectPass()); + return Error::success(); +} + +void X86CodeGenPassBuilder::addILPOpts(AddMachinePass &addPass) const { + addPass(EarlyIfConverterPass()); + if (EnableMachineCombinerPass) + addPass(MachineCombinerPass()); + addPass(X86CmovConverterDummyPass()); +} + +void X86CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const { + // Only add this pass for 32-bit x86 Windows. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows() && TT.getArch() == Triple::x86) + addPass(X86WinEHStatePass()); +} + +void X86CodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const { + if (TM.getOptLevel() != CodeGenOptLevel::None) { + addPass(LiveRangeShrinkPass()); + addPass(X86FixupSetCCPass()); + addPass(X86OptimizeLEAsPass()); + addPass(X86CallFrameOptimizationPass()); + addPass(X86AvoidStoreForwardingBlocksPass()); + } + + addPass(X86SpeculativeLoadHardeningPass()); + addPass(X86FlagsCopyLoweringDummyPass()); // TODO: port to NPM and rename + addPass(X86DynAllocaExpanderPass()); + + if (getOptLevel() != CodeGenOptLevel::None) + addPass(X86PreTileConfigPass()); + else + addPass(X86FastPreTileConfigPass()); +} + +Error X86CodeGenPassBuilder::addPostFastRegAllocRewrite( + AddMachinePass &addPass) const { + addPass(X86FastTileConfigPass()); + return Error::success(); +} + +void X86CodeGenPassBuilder::addMachineSSAOptimization( + AddMachinePass &addPass) const { + addPass(X86DomainReassignmentPass()); + CodeGenPassBuilder::addMachineSSAOptimization(addPass); +} + +void X86CodeGenPassBuilder::addPostRegAlloc(AddMachinePass &addPass) const { + addPass(X86LowerTileCopyPass()); + addPass(X86FloatingPointStackifierPass()); + // When -O0 is enabled, the Load Value Injection Hardening pass will fall back + // to using the Speculative Execution Side Effect Suppression pass for + // mitigation. This is to prevent slow downs due to + // analyses needed by the LVIHardening pass when compiling at -O0. + if (getOptLevel() != CodeGenOptLevel::None) + addPass(X86LoadValueInjectionRetHardeningPass()); +} + +void X86CodeGenPassBuilder::addPreSched2(AddMachinePass &addPass) const { + addPass(X86ExpandPseudoPass()); + addPass(MachineKCFIPass()); +} + +void X86CodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const { + if (getOptLevel() != CodeGenOptLevel::None) { + addPass(X86ExecutionDomainFixPass()); + addPass(BreakFalseDepsPass()); + } + + addPass(X86IndirectBranchTrackingPass()); + + addPass(X86IssueVZeroUpperPass()); + + if (getOptLevel() != CodeGenOptLevel::None) { + addPass(X86FixupBWInstsPass()); + addPass(X86PadShortFunctionsPass()); + addPass(X86FixupLEAsPass()); + addPass(X86FixupInstTuningPass()); + addPass(X86FixupVectorConstantsPass()); + } + addPass(X86EvexToVexInstsPass()); + addPass(X86DiscriminateMemOpsPass()); + addPass(X86InsertPrefetchPass()); + addPass(X86InsertX87waitPass()); +} + +void X86CodeGenPassBuilder::addPrecedingEmitPass( + AddMachinePass &addPass) const { + const Triple &TT = TM.getTargetTriple(); + const MCAsmInfo *MAI = TM.getMCAsmInfo(); + + // The X86 Speculative Execution Pass must run after all control + // flow graph modifying passes. As a result it was listed to run right before + // the X86 Retpoline Thunks pass. The reason it must run after control flow + // graph modifications is that the model of LFENCE in LLVM has to be updated + // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the + // placement of this pass was hand checked to ensure that the subsequent + // passes don't move the code around the LFENCEs in a way that will hurt the + // correctness of this pass. This placement has been shown to work based on + // hand inspection of the codegen output. + addPass(X86SpeculativeExecutionSideEffectSuppressionPass()); + addPass(X86IndirectThunksPass()); + addPass(X86ReturnThunksPass()); + + // Insert extra int3 instructions after trailing call instructions to avoid + // issues in the unwinder. + if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) + addPass(X86AvoidTrailingCallPass()); + + // Verify basic block incoming and outgoing cfa offset and register values and + // correct CFA calculation rule where needed by inserting appropriate CFI + // instructions. + if (!TT.isOSDarwin() && + (!TT.isOSWindows() || + MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) + addPass(CFIInstrInserterPass()); + // Identify valid longjmp targets for Windows Control Flow Guard. + if (TT.isOSWindows()) { + // Identify valid longjmp targets for Windows Control Flow Guard. + addPass(CFGuardLongjmpPass()); + // Identify valid eh continuation targets for Windows EHCont Guard. + addPass(EHContGuardCatchretPass()); + } + addPass(X86LoadValueInjectionRetHardeningPass()); + + // Insert pseudo probe annotation for callsite profiling + addPass(PseudoProbeInserterPass()); + + // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms, + // also CALL_RVMARKER. + addPass(UnpackMachineBundlesPass([&TT](const MachineFunction &MF) { + // Only run bundle expansion if the module uses kcfi, or there are relevant + // ObjC runtime functions present in the module. + const Function &F = MF.getFunction(); + const Module *M = F.getParent(); + return M->getModuleFlag("kcfi") || + (TT.isOSDarwin() && + (M->getFunction("objc_retainAutoreleasedReturnValue") || + M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))); + })); +} + +Error X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized( + AddMachinePass &addPass) const { + if (Opt.RegAlloc != RegAllocType::Default && EnableTileRAPass) { + addPass(RAGreedyPass(onlyAllocateTileRegisters)); + addPass(X86TileConfigPass()); + } + return CodeGenPassBuilder::addRegAssignAndRewriteFast(addPass); +} + +void X86CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass, + CreateMCStreamer callback) const { + addPass(X86AsmPrinterPass(callback)); +} + +std::pair +X86CodeGenPassBuilder::getTargetPassNameFromLegacyName(StringRef Name) const { + std::pair Ret; + +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#include "X86PassRegistry.def" + + return Ret; +} + +Error X86TargetMachine::buildCodeGenPipeline( + ModulePassManager &MPM, MachineFunctionPassManager &MFPM, + MachineFunctionAnalysisManager &MFAM, raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, CodeGenFileType FileType, + CGPassBuilderOption Opts, PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.buildPipeline(MPM, MFPM, Out, DwoOut, FileType); +} + +std::pair +X86TargetMachine::getPassNameFromLegacyName(StringRef Name) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.getPassNameFromLegacyName(Name); +} + +bool X86CodeGenPassBuilder::parseTargetMIRPass(MachineFunctionPassManager &MFPM, + StringRef Name) const { +#define ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + MFPM.addPass(PASS_NAME()); \ + return false; \ + } + +#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#include "X86PassRegistry.def" +#undef ADD_PASS + return true; +} + +Error X86TargetMachine::parseMIRPipeline(MachineFunctionPassManager &MFPM, + StringRef PipelineText, + CGPassBuilderOption Opts, + MachineFunctionAnalysisManager &MFAM, + PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.parseMIRPipeline(MFPM, PipelineText); +} diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index 4836be4db0e8e..f4e8ea85b62c3 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -50,6 +50,18 @@ class X86TargetMachine final : public LLVMTargetMachine { // Set up the pass pipeline. TargetPassConfig *createPassConfig(PassManagerBase &PM) override; + Error buildCodeGenPipeline(ModulePassManager &, MachineFunctionPassManager &, + MachineFunctionAnalysisManager &, + raw_pwrite_stream &, raw_pwrite_stream *, + CodeGenFileType, CGPassBuilderOption, + PassInstrumentationCallbacks *) override; + + std::pair getPassNameFromLegacyName(StringRef) override; + + Error parseMIRPipeline(MachineFunctionPassManager &MFPM, StringRef, + CGPassBuilderOption, MachineFunctionAnalysisManager &, + PassInstrumentationCallbacks *) override; + TargetLoweringObjectFile *getObjFileLowering() const override { return TLOF.get(); } From a87204c052bca91cc0c3b41ac07403a429d37bff Mon Sep 17 00:00:00 2001 From: Yuanfang Chen <455423+yuanfang-chen@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:48:45 +0800 Subject: [PATCH 3/4] [NewPM][CodeGen] Add NPM support to llc --- llvm/include/llvm/IR/PassInstrumentation.h | 33 +++ llvm/include/llvm/Target/TargetMachine.h | 11 +- llvm/lib/CodeGen/TargetPassConfig.cpp | 19 +- llvm/lib/Passes/StandardInstrumentations.cpp | 7 +- llvm/tools/llc/CMakeLists.txt | 3 + llvm/tools/llc/NewPMDriver.cpp | 232 +++++++++++++++++++ llvm/tools/llc/NewPMDriver.h | 49 ++++ llvm/tools/llc/llc.cpp | 55 ++--- 8 files changed, 362 insertions(+), 47 deletions(-) create mode 100644 llvm/tools/llc/NewPMDriver.cpp create mode 100644 llvm/tools/llc/NewPMDriver.h diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h index 519a5e46b4373..3f70fcf180af8 100644 --- a/llvm/include/llvm/IR/PassInstrumentation.h +++ b/llvm/include/llvm/IR/PassInstrumentation.h @@ -84,6 +84,23 @@ class PassInstrumentationCallbacks { using AfterAnalysisFunc = void(StringRef, Any); using AnalysisInvalidatedFunc = void(StringRef, Any); using AnalysesClearedFunc = void(StringRef); + using StartStopFunc = bool(StringRef, Any); + + struct CodeGenStartStopInfo { + StringRef Start; + StringRef Stop; + + bool IsStopMachinePass = false; + + llvm::unique_function StartStopCallback; + + bool operator()(StringRef PassID, Any IR) { + return StartStopCallback(PassID, IR); + } + bool isStopMachineFunctionPass() const { return IsStopMachinePass; } + bool willCompleteCodeGenPipeline() const { return Stop.empty(); } + StringRef getStop() const { return Stop; } + }; public: PassInstrumentationCallbacks() = default; @@ -148,6 +165,17 @@ class PassInstrumentationCallbacks { AnalysesClearedCallbacks.emplace_back(std::move(C)); } + void registerStartStopInfo(CodeGenStartStopInfo &&C) { + StartStopInfo = std::move(C); + } + + bool isStartStopInfoRegistered() const { return StartStopInfo.has_value(); } + + CodeGenStartStopInfo &getStartStopInfo() { + assert(StartStopInfo.has_value() && "StartStopInfo is unregistered!"); + return *StartStopInfo; + } + /// Add a class name to pass name mapping for use by pass instrumentation. void addClassToPassName(StringRef ClassName, StringRef PassName); /// Get the pass name for a given pass class name. @@ -183,6 +211,8 @@ class PassInstrumentationCallbacks { /// These are run on analyses that have been cleared. SmallVector, 4> AnalysesClearedCallbacks; + /// For `llc` -start-* -stop-* options. + std::optional StartStopInfo; StringMap ClassToPassName; }; @@ -236,6 +266,9 @@ class PassInstrumentation { ShouldRun &= C(Pass.name(), llvm::Any(&IR)); } + if (Callbacks->StartStopInfo) + ShouldRun &= (*Callbacks->StartStopInfo)(Pass.name(), llvm::Any(&IR)); + if (ShouldRun) { for (auto &C : Callbacks->BeforeNonSkippedPassCallbacks) C(Pass.name(), llvm::Any(&IR)); diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index 4c29f25bedf41..a6f33869ecde0 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -461,8 +461,15 @@ class LLVMTargetMachine : public TargetMachine { } virtual std::pair getPassNameFromLegacyName(StringRef) { - llvm_unreachable( - "getPassNameFromLegacyName parseMIRPipeline is not overridden"); + llvm_unreachable("getPassNameFromLegacyName is not overridden"); + } + + virtual Error parseMIRPipeline(MachineFunctionPassManager &MFPM, + StringRef PipelineText, + CGPassBuilderOption Opts, + MachineFunctionAnalysisManager &MFAM, + PassInstrumentationCallbacks *PIC) { + llvm_unreachable("parseMIRPipeline is not overridden"); } /// Add passes to the specified pass manager to get machine code emitted with diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index faa5466b69e8b..293dda7f44bd7 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -516,6 +516,9 @@ static void registerPartialPipelineCallback(PassInstrumentationCallbacks &PIC, unsigned StopBeforeInstanceNum = 0; unsigned StopAfterInstanceNum = 0; + bool IsStopBeforeMachinePass = false; + bool IsStopAfterMachinePass = false; + std::tie(StartBefore, StartBeforeInstanceNum) = getPassNameAndInstanceNum(StartBeforeOpt); std::tie(StartAfter, StartAfterInstanceNum) = @@ -544,7 +547,15 @@ static void registerPartialPipelineCallback(PassInstrumentationCallbacks &PIC, report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") + Twine(StopAfterOptName) + Twine(" specified!")); - PIC.registerShouldRunOptionalPassCallback( + std::vector SpecialPasses = {"PassManager", "PassAdaptor", + "PrintMIRPass", "PrintModulePass"}; + + PassInstrumentationCallbacks::CodeGenStartStopInfo Info; + Info.Start = StartBefore.empty() ? StartAfter : StartBefore; + Info.Stop = StopBefore.empty() ? StopAfter : StopBefore; + + Info.IsStopMachinePass = IsStopBeforeMachinePass || IsStopAfterMachinePass; + Info.StartStopCallback = [=, EnableCurrent = StartBefore.empty() && StartAfter.empty(), EnableNext = std::optional(), StartBeforeCount = 0u, StartAfterCount = 0u, StopBeforeCount = 0u, @@ -575,8 +586,10 @@ static void registerPartialPipelineCallback(PassInstrumentationCallbacks &PIC, EnableCurrent = true; if (StopBeforePass && StopBeforeCount++ == StopBeforeInstanceNum) EnableCurrent = false; - return EnableCurrent; - }); + return EnableCurrent || isSpecialPass(P, SpecialPasses); + }; + + PIC.registerStartStopInfo(std::move(Info)); } void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC, diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp index fd1317e3eb256..bff8985edc964 100644 --- a/llvm/lib/Passes/StandardInstrumentations.cpp +++ b/llvm/lib/Passes/StandardInstrumentations.cpp @@ -1039,9 +1039,10 @@ void PrintPassInstrumentation::registerCallbacks( SpecialPasses.emplace_back("PassAdaptor"); } - PIC.registerBeforeSkippedPassCallback([this, SpecialPasses](StringRef PassID, - Any IR) { - assert(!isSpecialPass(PassID, SpecialPasses) && + PIC.registerBeforeSkippedPassCallback([this, SpecialPasses, + &PIC](StringRef PassID, Any IR) { + assert((!isSpecialPass(PassID, SpecialPasses) || + PIC.isStartStopInfoRegistered()) && "Unexpectedly skipping special pass"); print() << "Skipping pass: " << PassID << " on " << getIRName(IR) << "\n"; diff --git a/llvm/tools/llc/CMakeLists.txt b/llvm/tools/llc/CMakeLists.txt index 257d5b519f040..01825c6e4c64c 100644 --- a/llvm/tools/llc/CMakeLists.txt +++ b/llvm/tools/llc/CMakeLists.txt @@ -8,9 +8,11 @@ set(LLVM_LINK_COMPONENTS CodeGen CodeGenTypes Core + IRPrinter IRReader MC MIRParser + Passes Remarks ScalarOpts SelectionDAG @@ -23,6 +25,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_tool(llc llc.cpp + NewPMDriver.cpp DEPENDS intrinsics_gen diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp new file mode 100644 index 0000000000000..833a8abbaa72b --- /dev/null +++ b/llvm/tools/llc/NewPMDriver.cpp @@ -0,0 +1,232 @@ +//===- NewPMDriver.cpp - Driver for llc using new PM ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file is just a split of the code that logically belongs in llc.cpp but +/// that includes the new pass manager headers. +/// +//===----------------------------------------------------------------------===// + +#include "NewPMDriver.h" +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/CodeGen/CodeGenPassBuilder.h" +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/MIRParser/MIRParser.h" +#include "llvm/CodeGen/MIRPrinter.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachinePassManager.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Target/CGPassBuilderOption.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" +#include "llvm/Transforms/Utils/Cloning.h" + +using namespace llvm; + +static cl::opt RegAlloc( + "regalloc-npm", cl::desc("Register allocator to use for new pass manager"), + cl::Hidden, cl::ValueOptional, cl::init(RegAllocType::Default), + cl::values( + clEnumValN(RegAllocType::Default, "default", + "pick register allocator based on -O option"), + clEnumValN(RegAllocType::Basic, "basic", "basic register allocator"), + clEnumValN(RegAllocType::Fast, "fast", "fast register allocator"), + clEnumValN(RegAllocType::Greedy, "greedy", "greedy register allocator"), + clEnumValN(RegAllocType::PBQP, "pbqp", "PBQP register allocator"))); + +static cl::opt + DebugPM("debug-pass-manager", cl::Hidden, + cl::desc("Print pass management debugging information")); + +bool LLCDiagnosticHandler::handleDiagnostics(const DiagnosticInfo &DI) { + if (DI.getKind() == llvm::DK_SrcMgr) { + const auto &DISM = cast(DI); + const SMDiagnostic &SMD = DISM.getSMDiag(); + + if (SMD.getKind() == SourceMgr::DK_Error) + *HasError = true; + + SMD.print(nullptr, errs()); + + // For testing purposes, we print the LocCookie here. + if (DISM.isInlineAsmDiag() && DISM.getLocCookie()) + WithColor::note() << "!srcloc = " << DISM.getLocCookie() << "\n"; + + return true; + } + + if (DI.getSeverity() == DS_Error) + *HasError = true; + + if (auto *Remark = dyn_cast(&DI)) + if (!Remark->isEnabled()) + return true; + + DiagnosticPrinterRawOStream DP(errs()); + errs() << LLVMContext::getDiagnosticMessagePrefix(DI.getSeverity()) << ": "; + DI.print(DP); + errs() << "\n"; + return true; +} + +static llvm::ExitOnError ExitOnErr; + +static void RunPasses(bool BOS, ToolOutputFile *Out, Module *M, + LLVMContext &Context, SmallString<0> &Buffer, + ModulePassManager *MPM, ModuleAnalysisManager *MAM, + MachineFunctionPassManager &MFPM, + MachineFunctionAnalysisManager &MFAM) { + assert(MAM && ""); + assert(M && "invalid input module!"); + + // Before executing passes, print the final values of the LLVM options. + cl::PrintOptionValues(); + + if (MPM) { + assert(MAM); + MPM->run(*M, *MAM); + } + + ExitOnErr(MFPM.run(*M, MFAM)); + + auto HasError = + ((const LLCDiagnosticHandler *)(Context.getDiagHandlerPtr()))->HasError; + if (*HasError) + exit(1); + + if (BOS) + Out->os() << Buffer; +} + +int llvm::compileModuleWithNewPM( + StringRef Arg0, std::unique_ptr M, std::unique_ptr MIR, + std::unique_ptr Target, std::unique_ptr Out, + std::unique_ptr DwoOut, LLVMContext &Context, + const TargetLibraryInfoImpl &TLII, bool NoVerify, + const std::vector &RunPassNames, CodeGenFileType FileType) { + + if (!RunPassNames.empty() && TargetPassConfig::hasLimitedCodeGenPipeline()) { + WithColor::warning(errs(), Arg0) + << "run-pass cannot be used with " + << TargetPassConfig::getLimitedCodeGenPipelineReason(" and ") << ".\n"; + return 1; + } + + LLVMTargetMachine &LLVMTM = static_cast(*Target); + + { + raw_pwrite_stream *OS = &Out->os(); + + // Manually do the buffering rather than using buffer_ostream, + // so we can memcmp the contents in CompileTwice mode in future. + SmallString<0> Buffer; + std::unique_ptr BOS; + if ((codegen::getFileType() != CodeGenFileType::AssemblyFile && + !Out->os().supportsSeeking())) { + BOS = std::make_unique(Buffer); + OS = BOS.get(); + } + + // Fetch options from TargetPassConfig + CGPassBuilderOption Opt = getCGPassBuilderOption(); + Opt.DisableVerify = NoVerify; + Opt.DebugPM = DebugPM; + Opt.RegAlloc = RegAlloc; + + PassInstrumentationCallbacks PIC; + StandardInstrumentations SI(Context, Opt.DebugPM); + SI.registerCallbacks(PIC); + registerCodeGenCallback(PIC, LLVMTM); + + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + PassBuilder PB(Target.get(), PipelineTuningOptions(), std::nullopt, &PIC); + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + FAM.registerPass([&] { return TargetLibraryAnalysis(TLII); }); + MAM.registerPass([&] { return MachineModuleAnalysis(&LLVMTM); }); + + MachineFunctionAnalysisManager MFAM(FAM, MAM); + + if (!RunPassNames.empty()) { + // Construct a custom pass pipeline that starts after instruction + // selection. + + if (!MIR) { + WithColor::warning(errs(), Arg0) << "run-pass is for .mir file only.\n"; + return 1; + } + + MachineFunctionPassManager MFPM; + ExitOnErr(LLVMTM.parseMIRPipeline(MFPM, llvm::join(RunPassNames, ","), + Opt, MFAM, &PIC)); + MFPM.addPass(PrintMIRPass(*OS)); + MFPM.addPass(FreeMachineFunctionPass()); + + auto &MMI = MFAM.getResult(*M); + if (MIR->parseMachineFunctions(*M, MMI)) + return 1; + + RunPasses(BOS.get(), Out.get(), M.get(), Context, Buffer, nullptr, + nullptr, MFPM, MFAM); + } else { + ModulePassManager MPM; + MachineFunctionPassManager MFPM; + + ExitOnErr(LLVMTM.buildCodeGenPipeline(MPM, MFPM, MFAM, *OS, + DwoOut ? &DwoOut->os() : nullptr, + FileType, Opt, &PIC)); + + // Add IR or MIR printing pass according the pass type. + if (PIC.isStartStopInfoRegistered()) { + auto &Info = PIC.getStartStopInfo(); + if (!Info.willCompleteCodeGenPipeline()) { + if (Info.isStopMachineFunctionPass()) + MFPM.addPass(PrintMIRPass(*OS)); + else + MPM.addPass(PrintModulePass(*OS)); + } + } + + RunPasses(BOS.get(), Out.get(), M.get(), Context, Buffer, &MPM, &MAM, + MFPM, MFAM); + } + } + + // Declare success. + Out->keep(); + if (DwoOut) + DwoOut->keep(); + + return 0; +} diff --git a/llvm/tools/llc/NewPMDriver.h b/llvm/tools/llc/NewPMDriver.h new file mode 100644 index 0000000000000..f122185520eb2 --- /dev/null +++ b/llvm/tools/llc/NewPMDriver.h @@ -0,0 +1,49 @@ +//===- NewPMDriver.h - Function to drive llc with the new PM ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// A single function which is called to drive the llc behavior for the new +/// PassManager. +/// +/// This is only in a separate TU with a header to avoid including all of the +/// old pass manager headers and the new pass manager headers into the same +/// file. Eventually all of the routines here will get folded back into +/// llc.cpp. +/// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TOOLS_LLC_NEWPMDRIVER_H +#define LLVM_TOOLS_LLC_NEWPMDRIVER_H + +#include "llvm/IR/DiagnosticHandler.h" +#include "llvm/Support/CodeGen.h" +#include +#include + +namespace llvm { +class Module; +class TargetLibraryInfoImpl; +class TargetMachine; +class ToolOutputFile; +class LLVMContext; +class MIRParser; + +struct LLCDiagnosticHandler : public DiagnosticHandler { + bool *HasError; + LLCDiagnosticHandler(bool *HasErrorPtr) : HasError(HasErrorPtr) {} + bool handleDiagnostics(const DiagnosticInfo &DI) override; +}; + +int compileModuleWithNewPM( + StringRef Arg0, std::unique_ptr M, std::unique_ptr MIR, + std::unique_ptr Target, std::unique_ptr Out, + std::unique_ptr DwoOut, LLVMContext &Context, + const TargetLibraryInfoImpl &TLII, bool NoVerify, + const std::vector &RunPassNames, CodeGenFileType FileType); +} // namespace llvm + +#endif diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp index 8d906cf372878..9befd1f25bc5b 100644 --- a/llvm/tools/llc/llc.cpp +++ b/llvm/tools/llc/llc.cpp @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +#include "NewPMDriver.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -186,6 +187,8 @@ static cl::opt RemarksFormat( cl::desc("The format used for serializing remarks (default: YAML)"), cl::value_desc("format"), cl::init("yaml")); +static cl::opt EnableNewPassManager( + "enable-new-pm", cl::desc("Enable the new pass manager"), cl::init(false)); static cl::opt TryUseNewDbgInfoFormat( "try-experimental-debuginfo-iterators", cl::desc("Enable debuginfo iterator positions, if they're built in"), @@ -306,41 +309,6 @@ static std::unique_ptr GetOutputStream(const char *TargetName, return FDOut; } -struct LLCDiagnosticHandler : public DiagnosticHandler { - bool *HasError; - LLCDiagnosticHandler(bool *HasErrorPtr) : HasError(HasErrorPtr) {} - bool handleDiagnostics(const DiagnosticInfo &DI) override { - if (DI.getKind() == llvm::DK_SrcMgr) { - const auto &DISM = cast(DI); - const SMDiagnostic &SMD = DISM.getSMDiag(); - - if (SMD.getKind() == SourceMgr::DK_Error) - *HasError = true; - - SMD.print(nullptr, errs()); - - // For testing purposes, we print the LocCookie here. - if (DISM.isInlineAsmDiag() && DISM.getLocCookie()) - WithColor::note() << "!srcloc = " << DISM.getLocCookie() << "\n"; - - return true; - } - - if (DI.getSeverity() == DS_Error) - *HasError = true; - - if (auto *Remark = dyn_cast(&DI)) - if (!Remark->isEnabled()) - return true; - - DiagnosticPrinterRawOStream DP(errs()); - errs() << LLVMContext::getDiagnosticMessagePrefix(DI.getSeverity()) << ": "; - DI.print(DP); - errs() << "\n"; - return true; - } -}; - // main - Entry point for the llc compiler. // int main(int argc, char **argv) { @@ -651,16 +619,12 @@ static int compileModule(char **argv, LLVMContext &Context) { reportError(EC.message(), SplitDwarfOutputFile); } - // Build up all of the passes that we want to do to the module. - legacy::PassManager PM; - // Add an appropriate TargetLibraryInfo pass for the module's triple. TargetLibraryInfoImpl TLII(Triple(M->getTargetTriple())); // The -disable-simplify-libcalls flag actually disables all builtin optzns. if (DisableSimplifyLibCalls) TLII.disableAllFunctions(); - PM.add(new TargetLibraryInfoWrapperPass(TLII)); // Verify module immediately to catch problems before doInitialization() is // called on any passes. @@ -676,6 +640,19 @@ static int compileModule(char **argv, LLVMContext &Context) { WithColor::warning(errs(), argv[0]) << ": warning: ignoring -mc-relax-all because filetype != obj"; + bool RunPassNone = + !getRunPassNames().empty() && getRunPassNames().at(0) == "none"; + if (EnableNewPassManager && !RunPassNone) { + return compileModuleWithNewPM(argv[0], std::move(M), std::move(MIR), + std::move(Target), std::move(Out), + std::move(DwoOut), Context, TLII, NoVerify, + getRunPassNames(), codegen::getFileType()); + } + + // Build up all of the passes that we want to do to the module. + legacy::PassManager PM; + PM.add(new TargetLibraryInfoWrapperPass(TLII)); + { raw_pwrite_stream *OS = &Out->os(); From 42a07f27e42f77fb2417f830c9551409ce1585fb Mon Sep 17 00:00:00 2001 From: PaperChalice Date: Thu, 9 Nov 2023 09:35:40 +0800 Subject: [PATCH 4/4] [NPM][llc] add NPM tests --- .../Generic/llc-start-stop-instance-errors.ll | 2 + .../CodeGen/Generic/new-pm/llc-start-stop.ll | 47 +++++ llvm/test/CodeGen/X86/new-pm/O0-pipeline.ll | 80 +++++++++ .../X86/new-pm/llc-start-stop-instance.ll | 35 ++++ llvm/test/CodeGen/X86/new-pm/opt-pipeline.ll | 166 ++++++++++++++++++ 5 files changed, 330 insertions(+) create mode 100644 llvm/test/CodeGen/Generic/new-pm/llc-start-stop.ll create mode 100644 llvm/test/CodeGen/X86/new-pm/O0-pipeline.ll create mode 100644 llvm/test/CodeGen/X86/new-pm/llc-start-stop-instance.ll create mode 100644 llvm/test/CodeGen/X86/new-pm/opt-pipeline.ll diff --git a/llvm/test/CodeGen/Generic/llc-start-stop-instance-errors.ll b/llvm/test/CodeGen/Generic/llc-start-stop-instance-errors.ll index 76cc8b681b6a7..8cad70b5d9ede 100644 --- a/llvm/test/CodeGen/Generic/llc-start-stop-instance-errors.ll +++ b/llvm/test/CodeGen/Generic/llc-start-stop-instance-errors.ll @@ -1,4 +1,6 @@ ; RUN: not --crash llc -debug-pass=Structure -stop-after=dead-mi-elimination,arst %s -o /dev/null 2>&1 \ ; RUN: | FileCheck -check-prefix=NOT-NUM %s +; RUN: not --crash llc -enable-new-pm -debug-pass-manager -stop-after=dead-mi-elimination,arst %s -o /dev/null 2>&1 \ +; RUN: | FileCheck -check-prefix=NOT-NUM %s ; NOT-NUM: LLVM ERROR: invalid pass instance specifier dead-mi-elimination,arst diff --git a/llvm/test/CodeGen/Generic/new-pm/llc-start-stop.ll b/llvm/test/CodeGen/Generic/new-pm/llc-start-stop.ll new file mode 100644 index 0000000000000..5245563763d81 --- /dev/null +++ b/llvm/test/CodeGen/Generic/new-pm/llc-start-stop.ll @@ -0,0 +1,47 @@ +; RUN: llc < %s -enable-new-pm -debug-pass-manager -stop-after=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-AFTER +; STOP-AFTER: Running pass: VerifierPass +; STOP-AFTER-NEXT: Running analysis: VerifierAnalysis +; STOP-AFTER-NEXT: Skipping pass: + +; RUN: llc < %s -enable-new-pm -debug-pass-manager -stop-before=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=STOP-BEFORE +; STOP-BEFORE: Running pass: AtomicExpandPass +; STOP-BEFORE-NEXT: Skipping pass: + +; RUN: llc < %s -enable-new-pm -debug-pass-manager -start-after=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=START-AFTER +; START-AFTER: Skipping pass: VerifierPass +; START-AFTER-NEXT: Running pass: + +; RUN: llc < %s -enable-new-pm -debug-pass-manager -start-before=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=START-BEFORE +; START-BEFORE-NOT: Running pass: +; START-BEFORE: Running pass: VerifierPass + +; RUN: not --crash llc < %s -enable-new-pm -start-before=nonexistent -o /dev/null 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NONEXISTENT-START-BEFORE +; RUN: not --crash llc < %s -enable-new-pm -stop-before=nonexistent -o /dev/null 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NONEXISTENT-STOP-BEFORE +; RUN: not --crash llc < %s -enable-new-pm -start-after=nonexistent -o /dev/null 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NONEXISTENT-START-AFTER +; RUN: not --crash llc < %s -enable-new-pm -stop-after=nonexistent -o /dev/null 2>&1 \ +; RUN: | FileCheck %s -check-prefix=NONEXISTENT-STOP-AFTER +; NONEXISTENT-START-BEFORE: "nonexistent" pass could not be found. +; NONEXISTENT-STOP-BEFORE: "nonexistent" pass could not be found. +; NONEXISTENT-START-AFTER: "nonexistent" pass could not be found. +; NONEXISTENT-STOP-AFTER: "nonexistent" pass could not be found. + +; RUN: not --crash llc < %s -enable-new-pm -start-before=verify -start-after=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=DOUBLE-START +; RUN: not --crash llc < %s -enable-new-pm -stop-before=verify -stop-after=verify \ +; RUN: -o /dev/null 2>&1 | FileCheck %s -check-prefix=DOUBLE-STOP +; DOUBLE-START: start-before and start-after specified! +; DOUBLE-STOP: stop-before and stop-after specified! + +define void @f() { + br label %b +b: + br label %b + ret void +} diff --git a/llvm/test/CodeGen/X86/new-pm/O0-pipeline.ll b/llvm/test/CodeGen/X86/new-pm/O0-pipeline.ll new file mode 100644 index 0000000000000..8e71c3e1feb5c --- /dev/null +++ b/llvm/test/CodeGen/X86/new-pm/O0-pipeline.ll @@ -0,0 +1,80 @@ +; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each +; pass. Ignore it with 'grep -v'. +; RUN: llc -mtriple=x86_64-- -O0 -debug-pass-manager -enable-new-pm < %s \ +; RUN: -o /dev/null 2>&1 | grep -v 'Verify generated machine code' | FileCheck %s + +; REQUIRES: asserts + +; CHECK-LABEL: Running pass: RequireAnalysisPass{{.*}} +; CHECK-NEXT: Running analysis: ProfileSummaryAnalysis on [module] +; CHECK-NEXT: Running pass: PreISelIntrinsicLoweringPass on [module] +; CHECK-NEXT: Running analysis: {{.*InnerAnalysisManagerProxy.*}} on [module] +; CHECK-NEXT: Running pass: ExpandLargeDivRemPass on f (1 instruction) +; CHECK-NEXT: Running pass: ExpandLargeFpConvertPass on f (1 instruction) +; CHECK-NEXT: Running pass: AtomicExpandPass on f (1 instruction) +; CHECK-NEXT: Running pass: VerifierPass on f (1 instruction) +; CHECK-NEXT: Running analysis: VerifierAnalysis on f +; CHECK-NEXT: Running pass: GCLoweringPass on f (1 instruction) +; CHECK-NEXT: Running pass: ShadowStackGCLoweringPass on f (1 instruction) +; CHECK-NEXT: Running pass: LowerConstantIntrinsicsPass on f (1 instruction) +; CHECK-NEXT: Running analysis: TargetLibraryAnalysis on f +; CHECK-NEXT: Running pass: UnreachableBlockElimPass on f (1 instruction) +; CHECK-NEXT: Running pass: ExpandVectorPredicationPass on f (1 instruction) +; CHECK-NEXT: Running analysis: TargetIRAnalysis on f +; CHECK-NEXT: Running pass: ScalarizeMaskedMemIntrinPass on f (1 instruction) +; CHECK-NEXT: Running pass: ExpandReductionsPass on f (1 instruction) +; CHECK-NEXT: Running pass: IndirectBrExpandPass on f (1 instruction) +; CHECK-NEXT: Running pass: DwarfEHPreparePass on f (1 instruction) +; CHECK-NEXT: Running pass: CallBrPreparePass on f (1 instruction) +; CHECK-NEXT: Running pass: SafeStackPass on f (1 instruction) +; CHECK-NEXT: Running pass: StackProtectorPass on f (1 instruction) +; CHECK-NEXT: Running pass: VerifierPass on f (1 instruction) +; CHECK-NEXT: Running analysis: MachineModuleAnalysis on [module] +; CHECK-NEXT: Running pass: {{.*}}::X86ISelDagPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86GlobalBaseRegPass on f +; CHECK-NEXT: Running pass: FinalizeISelPass on f +; CHECK-NEXT: Running pass: MachineVerifierPass on f +; CHECK-NEXT: Running pass: LocalStackSlotPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86SpeculativeLoadHardeningPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FlagsCopyLoweringDummyPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86DynAllocaExpanderPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FastPreTileConfigPass on f +; CHECK-NEXT: Running pass: PHIEliminationPass on f +; CHECK-NEXT: Running pass: TwoAddressInstructionPass on f +; CHECK-NEXT: Running pass: RegAllocPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FastTileConfigPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86LowerTileCopyPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FloatingPointStackifierPass on f +; CHECK-NEXT: Running pass: RemoveRedundantDebugValuesPass on f +; CHECK-NEXT: Running pass: FixupStatepointCallerSavedPass on f +; CHECK-NEXT: Running pass: PrologEpilogInserterPass on f +; CHECK-NEXT: Running pass: ExpandPostRAPseudosPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86ExpandPseudoPass on f +; CHECK-NEXT: Running pass: MachineKCFIPass on f +; CHECK-NEXT: Running pass: FEntryInserterPass on f +; CHECK-NEXT: Running pass: XRayInstrumentationPass on f +; CHECK-NEXT: Running pass: PatchableFunctionPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IndirectBranchTrackingPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IssueVZeroUpperPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86EvexToVexInstsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86DiscriminateMemOpsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86InsertPrefetchPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86InsertX87waitPass on f +; CHECK-NEXT: Running pass: FuncletLayoutPass on f +; CHECK-NEXT: Running pass: StackMapLivenessPass on f +; CHECK-NEXT: Running pass: LiveDebugValuesPass on f +; CHECK-NEXT: Running pass: MachineSanitizerBinaryMetadata on f +; CHECK-NEXT: Running pass: StackFrameLayoutAnalysisPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86SpeculativeExecutionSideEffectSuppressionPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IndirectThunksPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86ReturnThunksPass on f +; CHECK-NEXT: Running pass: CFIInstrInserterPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86LoadValueInjectionRetHardeningPass on f +; CHECK-NEXT: Running pass: PseudoProbeInserterPass on [module] +; CHECK-NEXT: Running pass: UnpackMachineBundlesPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86AsmPrinterPass on f +; CHECK-NEXT: Running pass: FreeMachineFunctionPass on f + +define void @f() { + ret void +} diff --git a/llvm/test/CodeGen/X86/new-pm/llc-start-stop-instance.ll b/llvm/test/CodeGen/X86/new-pm/llc-start-stop-instance.ll new file mode 100644 index 0000000000000..a6d5cf2fd8aff --- /dev/null +++ b/llvm/test/CodeGen/X86/new-pm/llc-start-stop-instance.ll @@ -0,0 +1,35 @@ +; RUN: llc -mtriple=x86_64-- -enable-new-pm -debug-pass-manager -stop-after=verify,1 \ +; RUN: %s -o /dev/null 2>&1 | FileCheck -check-prefix=STOP-AFTER-1 %s + +; RUN: llc -mtriple=x86_64-- -enable-new-pm -debug-pass-manager -stop-after=verify,0 \ +; RUN: %s -o /dev/null 2>&1 | FileCheck -check-prefix=STOP-AFTER-0 %s + +; RUN: llc -mtriple=x86_64-- -enable-new-pm -debug-pass-manager -stop-before=verify,1 \ +; RUN: %s -o /dev/null 2>&1 | FileCheck -check-prefix=STOP-BEFORE-1 %s + +; RUN: llc -mtriple=x86_64-- -enable-new-pm -debug-pass-manager -start-before=verify,1 \ +; RUN: %s -o /dev/null 2>&1 | FileCheck -check-prefix=START-BEFORE-1 %s + +; RUN: llc -mtriple=x86_64-- -enable-new-pm -debug-pass-manager -start-after=verify,1 \ +; RUN: %s -o /dev/null 2>&1 | FileCheck -check-prefix=START-AFTER-1 %s + + +; STOP-AFTER-1: Running pass: VerifierPass +; STOP-AFTER-1: Running pass: VerifierPass + +; STOP-AFTER-0-NOT: Running pass: VerifierPass +; STOP-AFTER-0: Running pass: VerifierPass +; STOP-AFTER-0-NOT: Running pass: VerifierPass + +; STOP-BEFORE-1: Running pass: VerifierPass +; STOP-BEFORE-1-NOT: Running pass: VerifierPass + +; START-BEFORE-1-NOT: Running pass: VerifierPass +; START-BEFORE-1: Running pass: VerifierPass +; START-BEFORE-1-NOT: Running pass: VerifierPass + +; START-AFTER-1-NOT: Running pass: VerifierPass + +define void @f() { + ret void +} diff --git a/llvm/test/CodeGen/X86/new-pm/opt-pipeline.ll b/llvm/test/CodeGen/X86/new-pm/opt-pipeline.ll new file mode 100644 index 0000000000000..7119facb8fc34 --- /dev/null +++ b/llvm/test/CodeGen/X86/new-pm/opt-pipeline.ll @@ -0,0 +1,166 @@ +; When EXPENSIVE_CHECKS are enabled, the machine verifier appears between each +; pass. Ignore it with 'grep -v'. +; RUN: llc -mtriple=x86_64-- -O1 -debug-pass-manager -enable-new-pm < %s \ +; RUN: -o /dev/null 2>&1 | FileCheck %s +; RUN: llc -mtriple=x86_64-- -O2 -debug-pass-manager -enable-new-pm < %s \ +; RUN: -o /dev/null 2>&1 | FileCheck %s +; RUN: llc -mtriple=x86_64-- -O3 -debug-pass-manager -enable-new-pm < %s \ +; RUN: -o /dev/null 2>&1 | FileCheck %s + +; REQUIRES: asserts + +; CHECK-LABEL: Running pass: RequireAnalysisPass{{.*}} +; CHECK-NEXT: Running analysis: ProfileSummaryAnalysis on [module] +; CHECK-NEXT: Running pass: PreISelIntrinsicLoweringPass on [module] +; CHECK-NEXT: Running analysis: {{.*InnerAnalysisManagerProxy.*}} on [module] +; CHECK-NEXT: Running pass: ExpandLargeDivRemPass on f (3 instructions) +; CHECK-NEXT: Running pass: ExpandLargeFpConvertPass on f (3 instructions) +; CHECK-NEXT: Running pass: AtomicExpandPass on f (3 instructions) +; CHECK-NEXT: Running pass: VerifierPass on f (3 instructions) +; CHECK-NEXT: Running analysis: VerifierAnalysis on f +; CHECK-NEXT: Running pass: LoopSimplifyPass on f (3 instructions) +; CHECK-NEXT: Running analysis: LoopAnalysis on f +; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on f +; CHECK-NEXT: Running analysis: AssumptionAnalysis on f +; CHECK-NEXT: Running analysis: TargetIRAnalysis on f +; CHECK-NEXT: Running pass: LCSSAPass on f (3 instructions) +; CHECK-NEXT: Running analysis: MemorySSAAnalysis on f +; CHECK-NEXT: Running analysis: AAManager on f +; CHECK-NEXT: Running analysis: TargetLibraryAnalysis on f +; CHECK-NEXT: Running analysis: BasicAA on f +; CHECK-NEXT: Running analysis: ScopedNoAliasAA on f +; CHECK-NEXT: Running analysis: TypeBasedAA on f +; CHECK-NEXT: Running analysis: {{.*OuterAnalysisManagerProxy.*}} on f +; CHECK-NEXT: Running analysis: ScalarEvolutionAnalysis on f +; CHECK-NEXT: Running analysis: {{.*InnerAnalysisManagerProxy.*}} on f +; CHECK-NEXT: Running pass: CanonicalizeFreezeInLoopsPass on b +; CHECK-NEXT: Running pass: LoopSimplifyPass on f (3 instructions) +; CHECK-NEXT: Running pass: LCSSAPass on f (3 instructions) +; CHECK-NEXT: Running pass: LoopStrengthReducePass on b +; CHECK-NEXT: Running analysis: IVUsersAnalysis on b +; CHECK-NEXT: Running pass: MergeICmpsPass on f (3 instructions) +; CHECK-NEXT: Running pass: ExpandMemCmpPass on f (3 instructions) +; CHECK-NEXT: Running pass: GCLoweringPass on f (3 instructions) +; CHECK-NEXT: Running pass: ShadowStackGCLoweringPass on f (3 instructions) +; CHECK-NEXT: Running pass: LowerConstantIntrinsicsPass on f (3 instructions) +; CHECK-NEXT: Running pass: UnreachableBlockElimPass on f (3 instructions) +; CHECK-NEXT: Clearing all analysis results for: +; CHECK-NEXT: Invalidating analysis: VerifierAnalysis on f +; CHECK-NEXT: Invalidating analysis: LoopAnalysis on f +; CHECK-NEXT: Invalidating analysis: MemorySSAAnalysis on f +; CHECK-NEXT: Invalidating analysis: ScalarEvolutionAnalysis on f +; CHECK-NEXT: Invalidating analysis: {{.*InnerAnalysisManagerProxy.*}} on f +; CHECK-NEXT: Running pass: ConstantHoistingPass on f (2 instructions) +; CHECK-NEXT: Running analysis: BlockFrequencyAnalysis on f +; CHECK-NEXT: Running analysis: BranchProbabilityAnalysis on f +; CHECK-NEXT: Running analysis: LoopAnalysis on f +; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on f +; CHECK-NEXT: Running pass: ReplaceWithVeclib on f (2 instructions) +; CHECK-NEXT: Running pass: PartiallyInlineLibCallsPass on f (2 instructions) +; CHECK-NEXT: Running pass: ExpandVectorPredicationPass on f (2 instructions) +; CHECK-NEXT: Running pass: ScalarizeMaskedMemIntrinPass on f (2 instructions) +; CHECK-NEXT: Running pass: ExpandReductionsPass on f (2 instructions) +; CHECK-NEXT: Running pass: TLSVariableHoistPass on f (2 instructions) +; CHECK-NEXT: Running pass: InterleavedAccessPass on f (2 instructions) +; CHECK-NEXT: Running pass: IndirectBrExpandPass on f (2 instructions) +; CHECK-NEXT: Running pass: CodeGenPreparePass on f (2 instructions) +; CHECK-NEXT: Running pass: DwarfEHPreparePass on f (2 instructions) +; CHECK-NEXT: Running pass: CallBrPreparePass on f (2 instructions) +; CHECK-NEXT: Running pass: SafeStackPass on f (2 instructions) +; CHECK-NEXT: Running pass: StackProtectorPass on f (2 instructions) +; CHECK-NEXT: Running pass: VerifierPass on f (2 instructions) +; CHECK-NEXT: Running analysis: VerifierAnalysis on f +; CHECK-NEXT: Running analysis: MachineModuleAnalysis on [module] +; CHECK-NEXT: Running pass: {{.*}}::X86ISelDagPass on f +; CHECK-NEXT: Running pass: {{.*}}::CleanupLocalDynamicTLSPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86GlobalBaseRegPass on f +; CHECK-NEXT: Running pass: FinalizeISelPass on f +; CHECK-NEXT: Running pass: MachineVerifierPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86DomainReassignmentPass on f +; CHECK-NEXT: Running pass: EarlyTailDuplicatePass on f +; CHECK-NEXT: Running pass: OptimizePHIsPass on f +; CHECK-NEXT: Running pass: StackColoringPass on f +; CHECK-NEXT: Running pass: LocalStackSlotPass on f +; CHECK-NEXT: Running pass: DeadMachineInstructionElimPass on f +; CHECK-NEXT: Running pass: EarlyIfConverterPass on f +; CHECK-NEXT: Running pass: MachineCombinerPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86CmovConverterDummyPass on f +; CHECK-NEXT: Running pass: EarlyMachineLICMPass on f +; CHECK-NEXT: Running pass: MachineCSEPass on f +; CHECK-NEXT: Running pass: MachineSinkingPass on f +; CHECK-NEXT: Running pass: PeepholeOptimizerPass on f +; CHECK-NEXT: Running pass: DeadMachineInstructionElimPass on f +; CHECK-NEXT: Running pass: LiveRangeShrinkPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FixupSetCCPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86OptimizeLEAsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86CallFrameOptimizationPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86AvoidStoreForwardingBlocksPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86SpeculativeLoadHardeningPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FlagsCopyLoweringDummyPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86DynAllocaExpanderPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86PreTileConfigPass on f +; CHECK-NEXT: Running pass: DetectDeadLanesPass on f +; CHECK-NEXT: Running pass: ProcessImplicitDefsPass on f +; CHECK-NEXT: Running pass: PHIEliminationPass on f +; CHECK-NEXT: Running pass: TwoAddressInstructionPass on f +; CHECK-NEXT: Running pass: RegisterCoalescerPass on f +; CHECK-NEXT: Running pass: RenameIndependentSubregsPass on f +; CHECK-NEXT: Running pass: MachineSchedulerPass on f +; CHECK-NEXT: Running pass: RegAllocPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FastTileConfigPass on f +; CHECK-NEXT: Running pass: MachineCopyPropagationPass on f +; CHECK-NEXT: Running pass: MachineLICMPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86LowerTileCopyPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FloatingPointStackifierPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86LoadValueInjectionRetHardeningPass on f +; CHECK-NEXT: Running pass: RemoveRedundantDebugValuesPass on f +; CHECK-NEXT: Running pass: FixupStatepointCallerSavedPass on f +; CHECK-NEXT: Running pass: PostRAMachineSinkingPass on f +; CHECK-NEXT: Running pass: ShrinkWrapPass on f +; CHECK-NEXT: Running pass: PrologEpilogInserterPass on f +; CHECK-NEXT: Running pass: BranchFolderPass on f +; CHECK-NEXT: Running pass: TailDuplicatePass on f +; CHECK-NEXT: Running pass: MachineLateInstrsCleanupPass on f +; CHECK-NEXT: Running pass: MachineCopyPropagationPass on f +; CHECK-NEXT: Running pass: ExpandPostRAPseudosPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86ExpandPseudoPass on f +; CHECK-NEXT: Running pass: MachineKCFIPass on f +; CHECK-NEXT: Running pass: PostRASchedulerPass on f +; CHECK-NEXT: Running pass: MachineBlockPlacementPass on f +; CHECK-NEXT: Running pass: FEntryInserterPass on f +; CHECK-NEXT: Running pass: XRayInstrumentationPass on f +; CHECK-NEXT: Running pass: PatchableFunctionPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86ExecutionDomainFixPass on f +; CHECK-NEXT: Running pass: BreakFalseDepsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IndirectBranchTrackingPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IssueVZeroUpperPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FixupBWInstsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86PadShortFunctionsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FixupLEAsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FixupInstTuningPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86FixupVectorConstantsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86EvexToVexInstsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86DiscriminateMemOpsPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86InsertPrefetchPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86InsertX87waitPass on f +; CHECK-NEXT: Running pass: FuncletLayoutPass on f +; CHECK-NEXT: Running pass: StackMapLivenessPass on f +; CHECK-NEXT: Running pass: LiveDebugValuesPass on f +; CHECK-NEXT: Running pass: MachineSanitizerBinaryMetadata on f +; CHECK-NEXT: Running pass: StackFrameLayoutAnalysisPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86SpeculativeExecutionSideEffectSuppressionPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86IndirectThunksPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86ReturnThunksPass on f +; CHECK-NEXT: Running pass: CFIInstrInserterPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86LoadValueInjectionRetHardeningPass on f +; CHECK-NEXT: Running pass: PseudoProbeInserterPass on [module] +; CHECK-NEXT: Running pass: UnpackMachineBundlesPass on f +; CHECK-NEXT: Running pass: {{.*}}::X86AsmPrinterPass on f +; CHECK-NEXT: Running pass: FreeMachineFunctionPass on f + +define void @f() { + br label %b +b: + br label %b + ret void +}